Rework proxy handling so that proxies can be specified as URLs.

As a side effect, remove a lot of duplicate and now redundant code.

2 files changed, 156 insertions, 206 deletions

diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 78e2de1..4a420ca 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -579,7 +579,7 @@ _ftp_default_port(void)
 {
     struct servent *se;
 	    
-    if ((se = getservbyname("ftp", "tcp")) != NULL)
+    if ((se = getservbyname(SCHEME_FTP, "tcp")) != NULL)
 	return ntohs(se->s_port);
     return FTP_DEFAULT_PORT;
 }
@@ -588,16 +588,16 @@ _ftp_default_port(void)
  * Log on to FTP server
  */
 static int
-_ftp_connect(char *host, int port, char *user, char *pwd, char *flags)
+_ftp_connect(struct url *url, struct url *purl, char *flags)
 {
-    int cd, e, pp = 0, direct, verbose;
+    int cd, e, direct, verbose;
 #ifdef INET6
     int af = AF_UNSPEC;
 #else
     int af = AF_INET;
 #endif
-    char *p, *q;
     const char *logname;
+    char *user, *pwd;
     char localhost[MAXHOSTNAMELEN];
     char pbuf[MAXHOSTNAMELEN + MAXLOGNAME + 1];
 
@@ -608,43 +608,17 @@ _ftp_connect(char *host, int port, char *user, char *pwd, char *flags)
     else if ((flags && strchr(flags, '6')))
 	af = AF_INET6;
 
+    if (direct)
+	purl = NULL;
+    
     /* check for proxy */
-    if (!direct && (p = getenv("FTP_PROXY")) != NULL && *p) {
-	char c = 0;
-
-#ifdef INET6
-	if (*p != '[' || (q = strchr(p + 1, ']')) == NULL ||
-	    (*++q != '\0' && *q != ':'))
-#endif
-	    q = strchr(p, ':');
-	if (q != NULL && *q == ':') {
-	    if (strspn(q+1, "0123456789") != strlen(q+1) || strlen(q+1) > 5) {
-		/* XXX we should emit some kind of warning */
-	    }
-	    pp = atoi(q+1);
-	    if (pp < 1 || pp > 65535) {
-		/* XXX we should emit some kind of warning */
-	    }
-	}
-	if (!pp)
-	    pp = _ftp_default_port();
-	if (q) {
-#ifdef INET6
-	    if (q > p && *p == '[' && *(q - 1) == ']') {
-		p++;
-		q--;
-	    }
-#endif
-	    c = *q;
-	    *q = 0;
-	}
-	cd = _fetch_connect(p, pp, af, verbose);
-	if (q)
-	    *q = c;
+    if (purl) {
+	/* XXX proxy authentication! */
+	cd = _fetch_connect(purl->host, purl->port, af, verbose);
     } else {
 	/* no proxy, go straight to target */
-	cd = _fetch_connect(host, port, af, verbose);
-	p = NULL;
+	cd = _fetch_connect(url->host, url->port, af, verbose);
+	purl = NULL;
     }
 
     /* check connection */
@@ -656,19 +630,23 @@ _ftp_connect(char *host, int port, char *user, char *pwd, char *flags)
     /* expect welcome message */
     if ((e = _ftp_chkerr(cd)) != FTP_SERVICE_READY)
 	goto fouch;
+
+    /* XXX FTP_AUTH, and maybe .netrc */
     
     /* send user name and password */
+    user = url->user;
     if (!user || !*user)
 	user = FTP_ANONYMOUS_USER;
-    if (p && port == FTP_DEFAULT_PORT)
-	e = _ftp_cmd(cd, "USER %s@%s", user, host);
-    else if (p)
-	e = _ftp_cmd(cd, "USER %s@%s@%d", user, host, port);
+    if (purl && url->port == FTP_DEFAULT_PORT)
+	e = _ftp_cmd(cd, "USER %s@%s", url->user, url->host);
+    else if (purl)
+	e = _ftp_cmd(cd, "USER %s@%s@%d", url->user, url->host, url->port);
     else
-	e = _ftp_cmd(cd, "USER %s", user);
+	e = _ftp_cmd(cd, "USER %s", url->user);
     
     /* did the server request a password? */
     if (e == FTP_NEED_PASSWORD) {
+	pwd = url->pwd;
 	if (!pwd || !*pwd)
 	    pwd = getenv("FTP_PASSWORD");
 	if (!pwd || !*pwd) {
@@ -734,7 +712,7 @@ _ftp_isconnected(struct url *url)
  * Check the cache, reconnect if no luck
  */
 static int
-_ftp_cached_connect(struct url *url, char *flags)
+_ftp_cached_connect(struct url *url, struct url *purl, char *flags)
 {
     int e, cd;
 
@@ -748,32 +726,40 @@ _ftp_cached_connect(struct url *url, char *flags)
     if (_ftp_isconnected(url)) {
 	e = _ftp_cmd(cached_socket, "NOOP");
 	if (e == FTP_OK || e == FTP_SYNTAX_ERROR)
-	    cd = cached_socket;
+	    return cached_socket;
     }
 
     /* connect to server */
-    if (cd == -1) {
-	cd = _ftp_connect(url->host, url->port, url->user, url->pwd, flags);
-	if (cd == -1)
-	    return -1;
-	if (cached_socket)
-	    _ftp_disconnect(cached_socket);
-	cached_socket = cd;
-	memcpy(&cached_host, url, sizeof *url);
-    }
-
+    if ((cd = _ftp_connect(url, purl, flags)) == -1)
+	return -1;
+    if (cached_socket)
+	_ftp_disconnect(cached_socket);
+    cached_socket = cd;
+    memcpy(&cached_host, url, sizeof *url);
     return cd;
 }
 
 /*
- * Check to see if we should use an HTTP proxy instead
+ * Check the proxy settings
  */
-static int
-_ftp_use_http_proxy(void)
+static struct url *
+_ftp_get_proxy(void)
 {
+    struct url *purl;
     char *p;
-
-    return ((p = getenv("HTTP_PROXY")) && *p && !getenv("FTP_PROXY"));
+    
+    if (((p = getenv("FTP_PROXY")) || (p = getenv("HTTP_PROXY"))) &&
+	*p && (purl = fetchParseURL(p)) != NULL) {
+	if (!*purl->scheme)
+	    strcpy(purl->scheme, SCHEME_FTP);
+	if (!purl->port)
+	    purl->port = _ftp_default_port();
+	if (strcasecmp(purl->scheme, SCHEME_FTP) == 0 ||
+	    strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+	    return purl;
+	fetchFreeURL(purl);
+    }
+    return NULL;
 }
 
 /*
@@ -782,13 +768,22 @@ _ftp_use_http_proxy(void)
 FILE *
 fetchXGetFTP(struct url *url, struct url_stat *us, char *flags)
 {
+    struct url *purl;
     int cd;
 
-    if (_ftp_use_http_proxy())
-	return fetchXGetHTTP(url, us, flags);
+    /* get the proxy URL, and check if we should use HTTP instead */
+    if (!strchr(flags, 'd') && (purl = _ftp_get_proxy()) != NULL) {
+	if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+	    return _http_request(url, "GET", us, purl, flags);
+    } else {
+	purl = NULL;
+    }
     
     /* connect to server */
-    if ((cd = _ftp_cached_connect(url, flags)) == NULL)
+    cd = _ftp_cached_connect(url, purl, flags);
+    if (purl)
+	fetchFreeURL(purl);
+    if (cd == NULL)
 	return NULL;
     
     /* change directory */
@@ -820,13 +815,23 @@ fetchGetFTP(struct url *url, char *flags)
 FILE *
 fetchPutFTP(struct url *url, char *flags)
 {
+    struct url *purl;
     int cd;
 
-    if (_ftp_use_http_proxy())
-	return fetchPutHTTP(url, flags);
+    /* get the proxy URL, and check if we should use HTTP instead */
+    if (!strchr(flags, 'd') && (purl = _ftp_get_proxy()) != NULL) {
+	if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+	    /* XXX HTTP PUT is not implemented, so try without the proxy */
+	    purl = NULL;
+    } else {
+	purl = NULL;
+    }
     
     /* connect to server */
-    if ((cd = _ftp_cached_connect(url, flags)) == NULL)
+    cd = _ftp_cached_connect(url, purl, flags);
+    if (purl)
+	fetchFreeURL(purl);
+    if (cd == NULL)
 	return NULL;
     
     /* change directory */
@@ -844,15 +849,30 @@ fetchPutFTP(struct url *url, char *flags)
 int
 fetchStatFTP(struct url *url, struct url_stat *us, char *flags)
 {
+    struct url *purl;
     int cd;
 
-    if (_ftp_use_http_proxy())
-	return fetchStatHTTP(url, us, flags);
+    /* get the proxy URL, and check if we should use HTTP instead */
+    if (!strchr(flags, 'd') && (purl = _ftp_get_proxy()) != NULL) {
+	if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {
+	    FILE *f;
+
+	    if ((f = _http_request(url, "HEAD", us, purl, flags)) == NULL)
+		return -1;
+	    fclose(f);
+	    return 0;
+	}
+    } else {
+	purl = NULL;
+    }
     
     /* connect to server */
-    if ((cd = _ftp_cached_connect(url, flags)) == NULL)
-	return -1;
-
+    cd = _ftp_cached_connect(url, purl, flags);
+    if (purl)
+	fetchFreeURL(purl);
+    if (cd == NULL)
+	return NULL;
+    
     /* change directory */
     if (_ftp_cwd(cd, url->doc) == -1)
 	return -1;
@@ -868,9 +888,6 @@ extern void warnx(char *, ...);
 struct url_ent *
 fetchListFTP(struct url *url, char *flags)
 {
-    if (_ftp_use_http_proxy())
-	return fetchListHTTP(url, flags);
-    
     warnx("fetchListFTP(): not implemented");
     return NULL;
 }
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 527be48..a386364 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -618,151 +618,66 @@ _http_default_port(char *scheme)
 
     if ((se = getservbyname(scheme, "tcp")) != NULL)
 	return ntohs(se->s_port);
-    if (strcasecmp(scheme, "ftp") == 0)
+    if (strcasecmp(scheme, SCHEME_FTP) == 0)
 	return FTP_DEFAULT_PORT;
-    if (strcasecmp(scheme, "http") == 0)
+    if (strcasecmp(scheme, SCHEME_HTTP) == 0)
 	return HTTP_DEFAULT_PORT;
     return 0;
 }
 
 /*
- * Connect to the specified HTTP proxy server.
- */
-static int
-_http_proxy_connect(char *proxy, int af, int verbose)
-{
-    char *hostname, *p;
-    int fd, port;
-
-    /* get hostname */
-    hostname = NULL;
-#ifdef INET6
-    /* host part can be an IPv6 address enclosed in square brackets */
-    if (*proxy == '[') {
-	if ((p = strchr(proxy, ']')) == NULL) {
-	    /* no terminating bracket */
-	    /* XXX should set an error code */
-	    goto ouch;
-	}
-	if (p[1] != '\0' && p[1] != ':') {
-	    /* garbage after address */
-	    /* XXX should set an error code */
-	    goto ouch;
-	}
-	if ((hostname = malloc(p - proxy)) == NULL) {
-	    errno = ENOMEM;
-	    _fetch_syserr();
-	    goto ouch;
-	}
-	strncpy(hostname, proxy + 1, p - proxy - 1);
-	hostname[p - proxy - 1] = '\0';
-	++p;
-    } else {
-#endif /* INET6 */
-	if ((p = strchr(proxy, ':')) == NULL)
-	    p = strchr(proxy, '\0');
-	if ((hostname = malloc(p - proxy + 1)) == NULL) {
-	    errno = ENOMEM;
-	    _fetch_syserr();
-	    goto ouch;
-	}
-	strncpy(hostname, proxy, p - proxy);
-	hostname[p - proxy] = '\0';
-#ifdef INET6
-    }
-#endif /* INET6 */
-    DEBUG(fprintf(stderr, "proxy name: [%s]\n", hostname));
-    
-    /* get port number */
-    port = 0;
-    if (*p == ':') {
-	++p;
-	if (strspn(p, "0123456789") != strlen(p) || strlen(p) > 5) {
-	    /* port number is non-numeric or too long */
-	    /* XXX should set an error code */
-	    goto ouch;
-	}
-	port = atoi(p);
-	if (port < 1 || port > 65535) {
-	    /* port number is out of range */
-	    /* XXX should set an error code */
-	    goto ouch;
-	}
-    }
-    
-    if (!port) {
-#if 0
-	/*
-	 * commented out, since there is currently no service name
-	 * for HTTP proxies
-	 */
-	struct servent *se;
-	
-	if ((se = getservbyname("xxxx", "tcp")) != NULL)
-	    port = ntohs(se->s_port);
-	else
-#endif
-	    port = 3128;
-    }
-    DEBUG(fprintf(stderr, "proxy port: %d\n", port));
-	
-    /* connect */
-    if ((fd = _fetch_connect(hostname, port, af, verbose)) == -1)
-	_fetch_syserr();
-    return fd;
-    
- ouch:
-    if (hostname)
-	free(hostname);
-    return -1;
-}
-
-/*
  * Connect to the correct HTTP server or proxy. 
  */
 static int
-_http_connect(struct url *URL, int *proxy, char *flags)
+_http_connect(struct url *URL, struct url *purl, char *flags)
 {
-    int direct, verbose;
+    int verbose;
     int af, fd;
-    char *p;
     
 #ifdef INET6
     af = AF_UNSPEC;
 #else
     af = AF_INET;
 #endif
-
-    direct = (flags && strchr(flags, 'd'));
+    
     verbose = (flags && strchr(flags, 'v'));
     if (flags && strchr(flags, '4'))
 	af = AF_INET;
+#ifdef INET6
     else if (flags && strchr(flags, '6'))
 	af = AF_INET6;
+#endif
+
+    if (purl) {
+	URL = purl;
+    } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) {
+	/* can't talk http to an ftp server */
+	/* XXX should set an error code */
+	return -1;
+    }
     
-    /* check port */
-    if (!URL->port)
-	URL->port = _http_default_port(URL->scheme);
+    if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
+	/* _fetch_connect() has already set an error code */
+	return -1;
+    return fd;
+}
+
+static struct url *
+_http_get_proxy()
+{
+    struct url *purl;
+    char *p;
     
-    if (!direct && (p = getenv("HTTP_PROXY")) != NULL && *p != '\0') {
-	/* attempt to connect to proxy server */
-	if ((fd = _http_proxy_connect(p, af, verbose)) == -1)
-	    return -1;
-	*proxy = 1;
-    } else {
-	/* if no proxy is configured, try direct */
-	if (strcasecmp(URL->scheme, "ftp") == 0) {
-	    /* can't talk http to an ftp server */
-	    /* XXX should set an error code */
-	    return -1;
-	}
-	if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
-	    /* _fetch_connect() has already set an error code */
-	    return -1;
-	*proxy = 0;
+    if ((p = getenv("HTTP_PROXY")) && (purl = fetchParseURL(p))) {
+	if (!*purl->scheme)
+	    strcpy(purl->scheme, SCHEME_HTTP);
+	if (!purl->port)
+	    purl->port = _http_default_port(SCHEME_HTTP);
+	if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+	    return purl;
+	fetchFreeURL(purl);
     }
-
-    return fd;
+    return NULL;
 }
 
 
@@ -773,11 +688,12 @@ _http_connect(struct url *URL, int *proxy, char *flags)
 /*
  * Send a request and process the reply
  */
-static FILE *
-_http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
+FILE *
+_http_request(struct url *URL, char *op, struct url_stat *us,
+	      struct url *purl, char *flags)
 {
     struct url *url, *new;
-    int chunked, need_auth, noredirect, proxy, verbose;
+    int chunked, direct, need_auth, noredirect, verbose;
     int code, fd, i, n;
     off_t offset, clength, length, size;
     time_t mtime;
@@ -789,9 +705,15 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
     char hbuf[MAXHOSTNAMELEN + 1];
 #endif
 
+    direct = (flags && strchr(flags, 'd'));
     noredirect = (flags && strchr(flags, 'A'));
     verbose = (flags && strchr(flags, 'v'));
 
+    if (direct && purl) {
+	fetchFreeURL(purl);
+	purl = NULL;
+    }
+    
     /* try the provided URL first */
     url = URL;
 
@@ -809,8 +731,12 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	size = -1;
 	mtime = 0;
     retry:
+	/* check port */
+	if (!url->port)
+	    url->port = _http_default_port(url->scheme);
+    
 	/* connect to server or proxy */
-	if ((fd = _http_connect(url, &proxy, flags)) == -1)
+	if ((fd = _http_connect(url, purl, flags)) == -1)
 	    goto ouch;
 
 	host = url->host;
@@ -825,7 +751,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	if (verbose)
 	    _fetch_info("requesting %s://%s:%d%s",
 			url->scheme, host, url->port, url->doc);
-	if (proxy) {
+	if (purl) {
 	    _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1",
 		      op, url->scheme, host, url->port, url->doc);
 	} else {
@@ -834,15 +760,18 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	}
 
 	/* proxy authorization */
-	if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0')
-	    _http_authorize(fd, "Proxy-Authorization", p);
+	if (purl) {
+	    if (*purl->user || *purl->pwd)
+		_http_basic_auth(fd, "Proxy-Authorization",
+				 purl->user, purl->pwd);
+	    else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0')
+		_http_authorize(fd, "Proxy-Authorization", p);
+	}
 	
 	/* server authorization */
 	if (need_auth) {
 	    if (*url->user || *url->pwd)
-		_http_basic_auth(fd, "Authorization",
-				 url->user ? url->user : "",
-				 url->pwd ? url->pwd : "");
+		_http_basic_auth(fd, "Authorization", url->user, url->pwd);
 	    else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0')
 		_http_authorize(fd, "Authorization", p);
 	    else {
@@ -1023,12 +952,16 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 
     if (url != URL)
 	fetchFreeURL(url);
+    if (purl)
+	fetchFreeURL(purl);
     
     return f;
 
  ouch:
     if (url != URL)
 	fetchFreeURL(url);
+    if (purl)
+	fetchFreeURL(purl);
     if (fd != -1)
 	close(fd);
     return NULL;
@@ -1045,7 +978,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 FILE *
 fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags)
 {
-    return _http_request(URL, "GET", us, flags);
+    return _http_request(URL, "GET", us, _http_get_proxy(), flags);
 }
 
 /*
@@ -1075,7 +1008,7 @@ fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags)
 {
     FILE *f;
     
-    if ((f = _http_request(URL, "HEAD", us, flags)) == NULL)
+    if ((f = _http_request(URL, "HEAD", us, _http_get_proxy(), flags)) == NULL)
 	return -1;
     fclose(f);
     return 0;