diff options
Diffstat (limited to 'lib/libfetch')
| -rw-r--r-- | lib/libfetch/Makefile | 44 | ||||
| -rw-r--r-- | lib/libfetch/README | 23 | ||||
| -rw-r--r-- | lib/libfetch/base64.c | 90 | ||||
| -rw-r--r-- | lib/libfetch/fetch.3 | 233 | ||||
| -rw-r--r-- | lib/libfetch/fetch.c | 189 | ||||
| -rw-r--r-- | lib/libfetch/fetch.h | 74 | ||||
| -rw-r--r-- | lib/libfetch/file.c | 49 | ||||
| -rw-r--r-- | lib/libfetch/ftp.c | 250 | ||||
| -rw-r--r-- | lib/libfetch/ftp.errors | 44 | ||||
| -rw-r--r-- | lib/libfetch/http.c | 338 | ||||
| -rw-r--r-- | lib/libfetch/http.errors | 41 |
11 files changed, 1375 insertions, 0 deletions
diff --git a/lib/libfetch/Makefile b/lib/libfetch/Makefile new file mode 100644 index 0000000..ad502d1 --- /dev/null +++ b/lib/libfetch/Makefile @@ -0,0 +1,44 @@ +LIB= fetch +CFLAGS+= -I${.CURDIR} -Wall +SRCS= fetch.c ftp.c http.c file.c base64.c +MAN3= fetch.3 +CLEANFILES+= ftperr.c httperr.c + +SHLIB_MAJOR= 1 +SHLIB_MINOR= 0 + +beforeinstall: + ${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 ${.CURDIR}/fetch.h \ + ${DESTDIR}/usr/include + +ftperr.c: ftp.errors + @echo "struct ftperr {" \ >> ${.TARGET} + @echo " const int num;" \ >> ${.TARGET} + @echo " const char *string;" \ >> ${.TARGET} + @echo "};" \ >> ${.TARGET} + @echo "static struct ftperr _ftp_errlist[] = {" \ >> ${.TARGET} + @cat ${.ALLSRC} \ + | grep -v ^# \ + | sort \ + | while read NUM STRING; do \ + echo " { $${NUM}, \"$${NUM} $${STRING}\" },"; \ + done >> ${.TARGET} + @echo " { 0, \"Unknown FTP error\" }" >> ${.TARGET} + @echo "};" >> ${.TARGET} + +httperr.c: http.errors + @echo "struct httperr {" \ >> ${.TARGET} + @echo " const int num;" \ >> ${.TARGET} + @echo " const char *string;" \ >> ${.TARGET} + @echo "};" \ >> ${.TARGET} + @echo "static struct httperr _http_errlist[] = {" \ >> ${.TARGET} + @cat ${.ALLSRC} \ + | grep -v ^# \ + | sort \ + | while read NUM STRING; do \ + echo " { $${NUM}, \"$${NUM} $${STRING}\" },"; \ + done >> ${.TARGET} + @echo " { 0, \"Unknown HTTP error\" }" >> ${.TARGET} + @echo "};" >> ${.TARGET} + +.include <bsd.lib.mk> diff --git a/lib/libfetch/README b/lib/libfetch/README new file mode 100644 index 0000000..30ddeae --- /dev/null +++ b/lib/libfetch/README @@ -0,0 +1,23 @@ +This is the new fetch(3) library, which is to replace the ftpio(3) +library and provide a new, unified backend for all fetch(1), +pkg_add(1) and sysinstall(8). + +Note that this is very much work in progress. It compiles (with a few +warnings), but there is much left to be implemented. Amongst other +items: + + * The man page needs work. Really. I mean it. Now. + + * ftp.c is not even half-written. + + * HTTP authentication doesn't work. I'm not sure if I bungled http.c + or fubared base64.c (which was ripped from MIT fetch(1)). + + * The library needs a decent interface for reporting errors. I've + started on something (sending back an error code in the url_t that + was sent in) but we're Not There (tm) yet. + +Comments, patches etc. of all kinds are welcome, but please don't +commit anything without talking to me first. + + -- Dag-Erling C. Smørgrav (des@FreeBSD.org) diff --git a/lib/libfetch/base64.c b/lib/libfetch/base64.c new file mode 100644 index 0000000..fcb628c --- /dev/null +++ b/lib/libfetch/base64.c @@ -0,0 +1,90 @@ +/*- + * Copyright 1997 Massachusetts Institute of Technology + * + * Permission to use, copy, modify, and distribute this software and + * its documentation for any purpose and without fee is hereby + * granted, provided that both the above copyright notice and this + * permission notice appear in all copies, that both the above + * copyright notice and this permission notice appear in all + * supporting documentation, and that the name of M.I.T. not be used + * in advertising or publicity pertaining to distribution of the + * software without specific, written prior permission. M.I.T. makes + * no representations about the suitability of this software for any + * purpose. It is provided "as is" without express or implied + * warranty. + * + * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS + * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT + * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: util.c,v 1.6 1998/02/20 05:08:53 jb Exp $ + */ + +#include <stdio.h> + +/* + * Not much left of the original MIT code, but it's still derived from it + * so I'll keep their copyright. This is taken from util.c in MIT fetch. + * + * -- DES 1998/05/22 + */ + +/* + * Implement the `base64' encoding as described in RFC 1521. + */ +static const char base64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +int +fprint64(FILE *f, const unsigned char *buf) +{ + int len = 0, l = 0; + unsigned int tmp; + + while (buf[len]) + len++; + + while (len >= 3) { + tmp = buf[0] << 16 | buf[1] << 8 | buf[2]; + fprintf(f, "%c%c%c%c", + base64[(tmp >> 18) & 077], + base64[(tmp >> 12) & 077], + base64[(tmp >> 6) & 077], + base64[tmp & 077]); + len -= 3; + buf += 3; + l += 4; + } + + /* RFC 1521 enumerates these three possibilities... */ + switch(len) { + case 2: + tmp = buf[0] << 16 | buf[1] << 8; + fprintf(f, "%c%c%c=", + base64[(tmp >> 18) & 077], + base64[(tmp >> 12) & 077], + base64[(tmp >> 6) & 077]); + l += 4; + break; + case 1: + tmp = buf[0] << 16; + fprintf(f, "%c%c==", + base64[(tmp >> 18) & 077], + base64[(tmp >> 12) & 077]); + l += 4; + break; + case 0: + break; + } + + return l; +} diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3 new file mode 100644 index 0000000..347af29 --- /dev/null +++ b/lib/libfetch/fetch.3 @@ -0,0 +1,233 @@ +.\" Copyright (c) 1998 Dag-Erling Coïdan Smørgrav +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $Id$ +.\" +.Dd July 1, 1998 +.Dt FETCH 3 +.Os +.Sh NAME +.Nm fetchParseURL , +.Nm fetchFreeURL , +.Nm fetchGetURL , +.Nm fetchPutURL , +.Nm fetchGetFile , +.Nm fetchPutFile , +.Nm fetchGetHTTP , +.Nm fetchPutHTTP , +.Nm fetchGetFTP , +.Nm fetchPutFTP +.Nd file transfer library +.Sh SYNOPSIS +.Fd #include <fetch.h> +.Ft url_t * +.Fn fetchParseURL "char *URL" "char *flags" +.Ft void +.Fn fetchFreeURL "url_t *u" +.Ft FILE * +.Fn fetchGetFile "url_t *u" "char *flags" +.Ft FILE * +.Fn fetchPutFile "url_t *u" "char *flags" +.Ft FILE * +.Fn fetchGetHTTP "url_t *u" "char *flags" +.Ft FILE * +.Fn fetchPutHTTP "url_t *u" "char *flags" +.Ft FILE * +.Fn fetchGetFTP "url_t *u" "char *flags" +.Ft FILE * +.Fn fetchPutFTP "url_t *u" "char *flags" +.Sh DESCRIPTION +These functions implement a high-level library for retrieving and +uploading files using Uniform Resource Locators (URLs). +.Pp +.Fn fetchParseURL +takes a URL in the form of a null-terminated string and splits it into +its components function according to the Common Internet Scheme Syntax +detailed in RFC1738. A regular expression which produces this syntax +is: +.Bd -literal + <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? +.Ed +.Pp +Note that some components of the URL are not necessarily relevant to +all URL schemes. For instance, the file scheme only needs the <scheme> +and <document> components. +.Pp +The pointer returned by +.Fn fetchParseURL +should be freed using +.Fn fetchFreeURL . +.Pp +.Fn fetchGetURL +and +.Fn fetchPutURL +constitute the recommended interface to the +.Nm fetch +library. They examine the URL passed to them to determine the transfer +method, and call the appropriate lower-level functions to perform the +actual transfer. The +.Fa flags +argument is a string of characters which specify transfer options. The +meaning of the individual flags is scheme-dependent, and is detailed +in the appropriate section below. +.Pp +All of the +.Fn fetchGetXXX +and +.Fn fetchPutXXX +functions return a pointer to a stream which can be used to read or +write data from or to the requested document, respectively. Not that +although the implementation details of the individual access methods +vary, it can generally be assumed that a stream returned by one of the +.Fn fetchGetXXX +functions is read-only, and that a stream returned by one of the +.Fn fetchPutXXX +functions is write-only. +.Sh FILE SCHEME +.Fn fetchGetFile +and +.Fn fetchPutFile +provide access to documents which are files in a locally mounted file +system. Only the <document> component of the URL is used. +.Pp +.Fn fetchGetFile +does not accept any flags. +.Pp +.Fn fetchPutFile +accepts the +.Fa a +(append to file) flag. If that flag is specified, the data written to +the stream returned by +.Fn fetchPutFile +will be appended to the previous contents of the file, instead of +replacing them. +.Sh FTP SCHEME +.Fn fetchGetFTP +and +.Fn fetchPutFTP +implement the FTP protocol as described in RFC959. +.Pp +If the +.Fa p +(passive) flag is specified, a passive (rather than active) connection +will be attempted. +.Pp +If no user name or password is given, the +.Nm fetch +library will attempt an anonymous login, with user name "ftp" and +password "ftp". +.Sh HTTP SCHEME +The +.Fn fetchGetHTTP +and +.Fn fetchPutHTTP +functions implement the HTTP/1.1 protocol. With a little luck, there's +even a chance that they comply with RFC2068. +.Pp +Since there seems to be no good way of implementing the HTTP PUT +method in a manner consistent with the rest of the +.Nm fetch +library, +.Fn fetchPutHTTP +is currently unimplemented. +.Sh RETURN VALUES +.Fn fetchParseURL +returns a pointer to a +.Fa url_t +structure containing the individual components of the URL. If it is +unable to allocate memory, or the URL is syntactically incorrect, +.Fn fetchParseURL +returns a NULL pointer. +.Pp +.Fn fetchFreeURL +does not return any value. +.Pp +All other functions return a stream pointer which may be used to +access the requested document. Upon failure of any kind, they return a +NULL pointer. +.Sh ENVIRONMENT +The FTP and HTTP functions use the +.Ev HTTP_PROXY +and +.Ev FTP_PROXY +environment variables, respectively, as the address of a proxy server +to use for transferring files. +.Sh SEE ALSO +.Xr fetch 1 , +.Xr ftpio 3 , +.Xr lots_of_other_stuff +.Rs +.%A T. Berners-Lee, L. Masinter & M. McCahill +.%D December 1994 +.%T Uniform Resource Locators (URL) +.%O RFC1738 +.Re +.Rs +.%A R. Fielding, J. Gettys, J. Mogul, H. Frystyk, T. Berners-Lee +.%D Januray 1997 +.%B Hypertext Transfer Protocol -- HTTP/1.1 +.%O RFC2068 +.Re +.Rs +.%A J. Postel, J. K. Reynolds +.%D October 1985 +.%B File Transfer Protocol +.%O RFC959 +.Re +.Sh DIAGNOSTICS +Add later. +.Sh NOTES +Some parts of the library are not yet implemented. The most notable +examples of this are +.Fn fetchPutHTTP +and proxy support for the FTP access method. +.Pp +I hate HTTP. +.Pp +Why does the \.Pp macro sometimes insert two blank lines instead of +one? +.Sh HISTORY +The +.Nm fetch +library first appeared in +.Fx 3.0 . +.Sh AUTHORS +The +.Nm fetch +library was mostly written by +.An Dag-Erling Coïdan Smørgrav Aq des@FreeBSD.org +with numerous suggestions from +.An Jordan K. Hubbard Aq jkh@FreeBSD.org +and other FreeBSD developers. +It incorporates the older +.Nm ftpio +library, which was originally written by +.Nm Poul-Henning Kamp Aq pkh@FreeBSD.org +and later turned inside out by +.An Jordan K. Hubbard Aq jkh@FreeBSD.org . +.Pp +This manual page was written by +.An Dag-Erling Coïdan Smørgrav Aq des@FreeBSD.org +.Sh BUGS +If I knew, I'd have fixed them. diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c new file mode 100644 index 0000000..99a2b9f --- /dev/null +++ b/lib/libfetch/fetch.c @@ -0,0 +1,189 @@ +/*- + * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id$ + */ + +#include <sys/param.h> + +#include <ctype.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "fetch.h" + +#ifndef NDEBUG +#define DEBUG(x) do x; while (0) +#else +#define DEBUG(x) do { } while (0) +#endif + + +/* get URL */ +FILE * +fetchGetURL(char *URL, char *flags) +{ + url_t *u; + FILE *f; + + /* parse URL */ + if ((u = fetchParseURL(URL)) == NULL) + return NULL; + + /* select appropriate function */ + if (strcasecmp(u->scheme, "file") == 0) + f = fetchGetFile(u, flags); + else if (strcasecmp(u->scheme, "http") == 0) + f = fetchGetHTTP(u, flags); + else if (strcasecmp(u->scheme, "ftp") == 0) + f = fetchGetFTP(u, flags); + else f = NULL; + + fetchFreeURL(u); + return f; +} + + +/* put URL */ +FILE * +fetchPutURL(char *URL, char *flags) +{ + url_t *u; + FILE *f; + + /* parse URL */ + if ((u = fetchParseURL(URL)) == NULL) + return NULL; + + /* select appropriate function */ + if (strcasecmp(u->scheme, "file") == 0) + f = fetchPutFile(u, flags); + else if (strcasecmp(u->scheme, "http") == 0) + f = fetchPutHTTP(u, flags); + else if (strcasecmp(u->scheme, "ftp") == 0) + f = fetchPutFTP(u, flags); + else f = NULL; + + fetchFreeURL(u); + return f; +} + +/* + * Split an URL into components. URL syntax is: + * method:[//[user[:pwd]@]host[:port]]/[document] + * This almost, but not quite, RFC1738 URL syntax. + */ +url_t * +fetchParseURL(char *URL) +{ + char *p, *q; + url_t *u; + int i; + + /* allocate url_t */ + if ((u = calloc(1, sizeof(url_t))) == NULL) + return NULL; + + /* scheme name */ + for (i = 0; *URL && (*URL != ':'); URL++) + if (i < URL_SCHEMELEN) + u->scheme[i++] = *URL; + if (!URL[0] || (URL[1] != '/')) + goto ouch; + else URL++; + if (URL[1] != '/') { + p = URL; + goto nohost; + } + else URL += 2; + + p = strpbrk(URL, "/@"); + if (*p == '@') { + /* username */ + for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) + if (i < URL_USERLEN) + u->user[i++] = *q; + + /* password */ + if (*q == ':') + for (q++, i = 0; (*q != ':') && (*q != '@'); q++) + if (i < URL_PWDLEN) + u->pwd[i++] = *q; + + p++; + } else p = URL; + + /* hostname */ + for (i = 0; *p && (*p != '/') && (*p != ':'); p++) + if (i < MAXHOSTNAMELEN) + u->host[i++] = *p; + + /* port */ + if (*p == ':') { + for (q = ++p; *q && (*q != '/'); q++) + if (isdigit(*q)) + u->port = u->port * 10 + (*q - '0'); + else return 0; /* invalid port */ + while (*p && (*p != '/')) + p++; + } + +nohost: + /* document */ + if (*p) + u->doc = strdup(p); + u->doc = strdup(*p ? p : "/"); + if (!u->doc) + goto ouch; + + DEBUG(fprintf(stderr, + "scheme: [\033[1m%s\033[m]\n" + "user: [\033[1m%s\033[m]\n" + "password: [\033[1m%s\033[m]\n" + "host: [\033[1m%s\033[m]\n" + "port: [\033[1m%d\033[m]\n" + "document: [\033[1m%s\033[m]\n", + u->scheme, u->user, u->pwd, + u->host, u->port, u->doc)); + + return u; + +ouch: + free(u); + return NULL; +} + +void +fetchFreeURL(url_t *u) +{ + if (u) { + if (u->doc) + free(u->doc); + free(u); + } +} diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h new file mode 100644 index 0000000..387d4c4 --- /dev/null +++ b/lib/libfetch/fetch.h @@ -0,0 +1,74 @@ +/*- + * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id$ + */ + +#ifndef _FETCH_H_INCLUDED +#define _FETCH_H_INCLUDED + +#include <sys/param.h> + +#define _LIBFETCH_VER "libfetch/1.0" + +#define URL_SCHEMELEN 16 +#define URL_USERLEN 256 +#define URL_PWDLEN 256 + +struct url_s { + char scheme[URL_SCHEMELEN+1]; + char user[URL_USERLEN+1]; + char pwd[URL_PWDLEN+1]; + char host[MAXHOSTNAMELEN+1]; + char *doc; + int port; + char *lasterr; +}; + +typedef struct url_s url_t; + +/* FILE-specific functions */ +FILE *fetchGetFile(url_t *, char *); +FILE *fetchPutFile(url_t *, char *); + +/* HTTP-specific functions */ +char *fetchContentType(FILE *f); +FILE *fetchGetHTTP(url_t *, char *); +FILE *fetchPutHTTP(url_t *, char *); + +/* FTP-specific functions */ +FILE *fetchGetFTP(url_t *, char *); +FILE *fetchPutFTP(url_t *, char *); + +/* Generic functions */ +url_t *fetchParseURL(char *URL); +void fetchFreeURL(url_t *u); +FILE *fetchGetURL(char *, char *); +FILE *fetchPutURL(char *, char *); + + +#endif diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c new file mode 100644 index 0000000..0c685ef --- /dev/null +++ b/lib/libfetch/file.c @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id$ + */ + +#include <stdio.h> +#include <string.h> + +#include "fetch.h" + +FILE * +fetchGetFile(url_t *u, char *flags) +{ + flags = flags; /* unused */ + return fopen(u->doc, "r"); +} + +FILE * +fetchPutFile(url_t *u, char *flags) +{ + if (strchr(flags, 'a')) + return fopen(u->doc, "a"); + else return fopen(u->doc, "w"); +} diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c new file mode 100644 index 0000000..14d43b9 --- /dev/null +++ b/lib/libfetch/ftp.c @@ -0,0 +1,250 @@ +/*- + * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id$ + */ + +/* + * Portions of this code were taken from ftpio.c: + * + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@login.dknet.dk> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * Major Changelog: + * + * Dag-Erling Coïdan Smørgrav + * 9 Jun 1998 + * + * Incorporated into libfetch + * + * Jordan K. Hubbard + * 17 Jan 1996 + * + * Turned inside out. Now returns xfers as new file ids, not as a special + * `state' of FTP_t + * + * $ftpioId: ftpio.c,v 1.30 1998/04/11 07:28:53 phk Exp $ + * + */ + +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> + +#include <ctype.h> +#include <netdb.h> +#include <stdio.h> +#include <string.h> + +#include "fetch.h" +#include "ftperr.c" + +#define FTP_ANONYMOUS_USER "ftp" +#define FTP_ANONYMOUS_PASSWORD "ftp" + +static url_t cached_host; +static FILE *cached_socket; +static int _ftp_errcode; + +static int +_ftp_isconnected(url_t *url) +{ + return (cached_socket + && (strcmp(url->host, cached_host.host) == 0) + && (strcmp(url->user, cached_host.user) == 0) + && (strcmp(url->pwd, cached_host.pwd) == 0) + && (url->port == cached_host.port)); +} + +/* + * Get server response, check that first digit is a '2' + */ +static int +_ftp_chkerr(FILE *s, char *e) +{ + char *line; + size_t len; + + do { + if (((line = fgetln(s, &len)) == NULL) || (len < 4)) + return -1; + } while (line[3] == '-'); + + if (!isdigit(line[0]) || !isdigit(line[1]) || !isdigit(line[2]) || (line[3] != ' ')) + return -1; + + _ftp_errcode = (line[0] - '0') * 100 + (line[1] - '0') * 10 + (line[2] - '0'); + + if (e) + *e = _ftp_errcode; + + return (line[0] == '2') - 1; +} + +/* + * Map error code to string + */ +static const char * +_ftp_errstring(int e) +{ + struct ftperr *p = _ftp_errlist; + + while ((p->num) && (p->num != e)) + p++; + + return p->string; +} + +/* + * Change remote working directory + */ +static int +_ftp_cwd(FILE *s, char *dir) +{ + fprintf(s, "CWD %s\n", dir); + if (ferror(s)) + return -1; + return _ftp_chkerr(s, NULL); /* expecting 250 */ +} + +/* + * Retrieve file + */ +static FILE * +_ftp_retr(FILE *s, char *file, int pasv) +{ + char *p; + + /* change directory */ + if (((p = strrchr(file, '/')) != NULL) && (p != file)) { + *p = 0; + if (_ftp_cwd(s, file) < 0) { + *p = '/'; + return NULL; + } + *p++ = '/'; + } else { + if (_ftp_cwd(s, "/") < 0) + return NULL; + } + + /* retrieve file; p now points to file name */ + return NULL; +} + + +/* + * XXX rewrite these + */ +#if 0 +FILE * +fetchGetFTP(url_t *url, char *flags) +{ + int retcode = 0; + static FILE *fp = NULL; + static char *prev_host = NULL; + FILE *fp2; + +#ifdef DEFAULT_TO_ANONYMOUS + if (!url->user[0]) { + strcpy(url->user, FTP_ANONYMOUS_USER); + strcpy(url->pwd, FTP_ANONYMOUS_PASSWORD); + } +#endif + + if (fp && prev_host) { + if (!strcmp(prev_host, url->host)) { + /* Try to use cached connection */ + fp2 = ftpGet(fp, url->doc, NULL); + if (!fp2) { + /* Connection timed out or was no longer valid */ + fclose(fp); + free(prev_host); + prev_host = NULL; + } + else + return fp2; + } + else { + /* It's a different host now, flush old */ + fclose(fp); + free(prev_host); + prev_host = NULL; + } + } + fp = ftpLogin(url->host, url->user, url->pwd, url->port, 0, &retcode); + if (fp) { + if (strchr(flags, 'p')) { + if (ftpPassive(fp, 1) != SUCCESS) + /* XXX what should we do? */ ; + } + fp2 = ftpGet(fp, url->doc, NULL); + if (!fp2) { + /* Connection timed out or was no longer valid */ + retcode = ftpErrno(fp); + fclose(fp); + fp = NULL; + } + else + prev_host = strdup(url->host); + return fp2; + } + return NULL; +} + +FILE * +fetchPutFTP(url_t *url, char *flags) +{ + static FILE *fp = NULL; + FILE *fp2; + int retcode = 0; + + if (fp) { /* Close previous managed connection */ + fclose(fp); + fp = NULL; + } + fp = ftpLogin(url->host, url->user, url->pwd, url->port, 0, &retcode); + if (fp) { + if (strchr(flags, 'p')) { + if (ftpPassive(fp, 1) != SUCCESS) + /* XXX what should we do? */ ; + } + fp2 = ftpPut(fp, url->doc); + if (!fp2) { + retcode = ftpErrno(fp); + fclose(fp); + fp = NULL; + } + return fp2; + } + return NULL; +} +#endif diff --git a/lib/libfetch/ftp.errors b/lib/libfetch/ftp.errors new file mode 100644 index 0000000..53a87f0 --- /dev/null +++ b/lib/libfetch/ftp.errors @@ -0,0 +1,44 @@ +# $Id: ftp.errors,v 1.3 1997/02/22 15:06:47 peter Exp $ +# +# This list is taken from RFC 959. +# It probably needs a going over. +# +110 Restart marker reply +120 Service ready in a few minutes +125 Data connection already open; transfer starting +150 File status okay; about to open data connection +200 Command okay +202 Command not implemented, superfluous at this site +211 System status, or system help reply +212 Directory status +213 File status +214 Help message +215 Set system type +220 Service ready for new user +221 Service closing control connection +225 Data connection open; no transfer in progress +226 Requested file action successful +227 Entering Passive Mode +230 User logged in, proceed +250 Requested file action okay, completed +257 File/directory created +331 User name okay, need password +332 Need account for login +350 Requested file action pending further information +421 Service not available, closing control connection +425 Can't open data connection +426 Connection closed; transfer aborted +450 File unavailable (e.g., file busy) +451 Requested action aborted: local error in processing +452 Insufficient storage space in system +500 Syntax error, command unrecognized +501 Syntax error in parameters or arguments +502 Command not implemented +503 Bad sequence of commands +504 Command not implemented for that parameter +530 Not logged in +532 Need account for storing files +550 File unavailable (e.g., file not found, no access) +551 Requested action aborted. Page type unknown +552 Exceeded storage allocation +553 File name not allowed diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c new file mode 100644 index 0000000..1cf9a9f --- /dev/null +++ b/lib/libfetch/http.c @@ -0,0 +1,338 @@ +/*- + * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id$ + */ + +#include <sys/param.h> +#include <sys/errno.h> +#include <sys/socket.h> +#include <sys/types.h> + +#include <netinet/in.h> + +#include <err.h> +#include <ctype.h> +#include <netdb.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "fetch.h" +#include "httperr.c" + +#ifndef NDEBUG +#define DEBUG(x) do x; while (0) +#else +#define DEBUG(x) do { } while (0) +#endif + +extern char *__progname; + +extern int fprint64(FILE *f, const unsigned char *buf); + +#define ENDL "\r\n" + +struct cookie +{ + FILE *real_f; +#define ENC_NONE 0 +#define ENC_CHUNKED 1 + int encoding; /* 1 = chunked, 0 = none */ +#define HTTPCTYPELEN 59 + char content_type[HTTPCTYPELEN+1]; + char *buf; + int b_cur, eof; + unsigned b_len, chunksize; +}; + +static int +_http_connect(char *host, int port) +{ + struct sockaddr_in sin; + struct hostent *he; + int fd; + + /* look up host name */ + if ((he = gethostbyname(host)) == NULL) + return -1; + + /* set up socket address structure */ + bzero(&sin, sizeof(sin)); + bcopy(he->h_addr, (char *)&sin.sin_addr, he->h_length); + sin.sin_family = he->h_addrtype; + sin.sin_port = htons(port); + + /* try to connect */ + if ((fd = socket(sin.sin_family, SOCK_STREAM, 0)) < 0) + return -1; + if (connect(fd, (struct sockaddr *)&sin, sizeof sin) < 0) { + close(fd); + return -1; + } + + return fd; +} + +static char * +_http_fillbuf(struct cookie *c) +{ + char *ln; + unsigned int len; + + if (c->eof) + return NULL; + + if (c->encoding == ENC_NONE) { + c->buf = fgetln(c->real_f, &(c->b_len)); + c->b_cur = 0; + } else if (c->encoding == ENC_CHUNKED) { + if (c->chunksize == 0) { + ln = fgetln(c->real_f, &len); + DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): new chunk: " + "%*.*s\033[m\n", (int)len-2, (int)len-2, ln)); + sscanf(ln, "%x", &(c->chunksize)); + if (!c->chunksize) { + DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): " + "end of last chunk\033[m\n")); + c->eof = 1; + return NULL; + } + DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): " + "new chunk: %X\033[m\n", c->chunksize)); + } + c->buf = fgetln(c->real_f, &(c->b_len)); + if (c->b_len > c->chunksize) + c->b_len = c->chunksize; + c->chunksize -= c->b_len; + c->b_cur = 0; + } + else return NULL; /* unknown encoding */ + return c->buf; +} + +static int +_http_readfn(struct cookie *c, char *buf, int len) +{ + int l, pos = 0; + while (len) { + /* empty buffer */ + if (!c->buf || (c->b_cur == c->b_len)) + if (!_http_fillbuf(c)) + break; + + l = c->b_len - c->b_cur; + if (len < l) l = len; + memcpy(buf + pos, c->buf + c->b_cur, l); + c->b_cur += l; + pos += l; + len -= l; + } + + if (ferror(c->real_f)) + return -1; + else return pos; +} + +static int +_http_writefn(struct cookie *c, const char *buf, int len) +{ + size_t r = fwrite(buf, 1, (size_t)len, c->real_f); + return r ? r : -1; +} + +static int +_http_closefn(struct cookie *c) +{ + int r = fclose(c->real_f); + free(c); + return (r == EOF) ? -1 : 0; +} + +char * +fetchContentType(FILE *f) +{ + /* + * We have no way of making sure this really *is* one of our cookies, + * so just check for a null pointer and hope for the best. + */ + return f->_cookie ? (((struct cookie *)f->_cookie)->content_type) : NULL; +} + +FILE * +fetchGetHTTP(url_t *URL, char *flags) +{ + int fd = -1, err, i, enc = ENC_NONE; + struct cookie *c; + char *ln, *p, *q; + FILE *f, *cf; + size_t len; + + /* allocate cookie */ + if ((c = calloc(1, sizeof(struct cookie))) == NULL) + return NULL; + + /* check port */ + if (!URL->port) + URL->port = 80; /* default HTTP port */ + + /* attempt to connect to proxy server */ + if (getenv("HTTP_PROXY")) { + char *px, host[MAXHOSTNAMELEN]; + int port = 3128; /* XXX I think 3128 is default... check? */ + size_t len; + + /* measure length */ + px = getenv("HTTP_PROXY"); + len = strcspn(px, ":"); + + /* get port (atoi is a little too tolerant perhaps?) */ + if (px[len] == ':') + port = atoi(px+len+1); + + /* get host name */ + if (len >= MAXHOSTNAMELEN) + len = MAXHOSTNAMELEN - 1; + strncpy(host, px, len); + host[len] = 0; + + /* connect */ + fd = _http_connect(host, port); + } + + /* if no proxy is configured or could be contacted, try direct */ + if (fd < 0) { + if ((fd = _http_connect(URL->host, URL->port)) < 0) + goto ouch; + } + + /* reopen as stream */ + if ((f = fdopen(fd, "r+")) == NULL) + goto ouch; + c->real_f = f; + + /* send request (proxies require absolute form, so use that) */ + fprintf(f, "GET http://%s:%d/%s HTTP/1.1" ENDL, + URL->host, URL->port, URL->doc); + + /* start sending headers away */ + if (URL->user[0] || URL->pwd[0]) { + fprintf(f, "Authorization: Basic "); + fprint64(f, (const unsigned char *)URL->user); + fputc(':', f); + fprint64(f, (const unsigned char *)URL->pwd); + fputs(ENDL, f); + } + fprintf(f, "Host: %s:%d" ENDL, URL->host, URL->port); + fprintf(f, "User-Agent: %s " _LIBFETCH_VER ENDL, __progname); + fprintf(f, "Connection: close" ENDL ENDL); + + /* get response */ + if ((ln = fgetln(f, &len)) == NULL) + goto fouch; + DEBUG(fprintf(stderr, "response: [\033[1m%*.*s\033[m]\n", + (int)len-2, (int)len-2, ln)); + + /* we can't use strchr() and friends since ln isn't NUL-terminated */ + p = ln; + while ((p < ln + len) && !isspace(*p)) + p++; + while ((p < ln + len) && !isdigit(*p)) + p++; + if (!isdigit(*p)) + goto fouch; + err = atoi(p); + DEBUG(fprintf(stderr, "code: [\033[1m%d\033[m]\n", err)); + + /* add code to handle redirects later */ + if (err != 200) + goto fouch; + + /* browse through header */ + while (1) { + if ((ln = fgetln(f, &len)) == NULL) + goto fouch; + if ((ln[0] == '\r') || (ln[0] == '\n')) + break; + DEBUG(fprintf(stderr, "header: [\033[1m%*.*s\033[m]\n", + (int)len-2, (int)len-2, ln)); +#define XFERENC "Transfer-Encoding:" + if (strncasecmp(ln, XFERENC, sizeof(XFERENC)-1) == 0) { + p = ln + sizeof(XFERENC) - 1; + while ((p < ln + len) && isspace(*p)) + p++; + for (q = p; (q < ln + len) && !isspace(*q); q++) + /* VOID */ ; + *q = 0; + if (strcasecmp(p, "chunked") == 0) + enc = ENC_CHUNKED; + DEBUG(fprintf(stderr, "xferenc: [\033[1m%s\033[m]\n", p)); +#undef XFERENC +#define CONTTYPE "Content-Type:" + } else if (strncasecmp(ln, CONTTYPE, sizeof(CONTTYPE)-1) == 0) { + p = ln + sizeof(CONTTYPE) - 1; + while ((p < ln + len) && isspace(*p)) + p++; + for (i = 0; p < ln + len; p++) + if (i < HTTPCTYPELEN) + c->content_type[i++] = *p; + do c->content_type[i--] = 0; while (isspace(c->content_type[i])); + DEBUG(fprintf(stderr, "conttype: [\033[1m%s\033[m]\n", + c->content_type)); +#undef CONTTYPE + } + } + + /* only body remains */ + c->encoding = enc; + cf = funopen(c, + (int (*)(void *, char *, int))_http_readfn, + (int (*)(void *, const char *, int))_http_writefn, + (fpos_t (*)(void *, fpos_t, int))NULL, + (int (*)(void *))_http_closefn); + if (cf == NULL) + goto fouch; + return cf; + +ouch: + if (fd >= 0) + close(fd); + free(c); + return NULL; +fouch: + fclose(f); + free(c); + return NULL; +} + +FILE * +fetchPutHTTP(url_t *URL, char *flags) +{ + warnx("fetchPutHTTP(): not implemented"); + return NULL; +} diff --git a/lib/libfetch/http.errors b/lib/libfetch/http.errors new file mode 100644 index 0000000..c0e3539 --- /dev/null +++ b/lib/libfetch/http.errors @@ -0,0 +1,41 @@ +# $Id$ +# +# This list is taken from RFC 2068. +# +100 Continue +101 Switching Protocols +200 OK +201 Created +202 Accepted +203 Non-Authoritative Information +204 No Content +205 Reset Content +206 Partial Content +300 Multiple Choices +301 Moved Permanently +302 Moved Temporarily +303 See Other +304 Not Modified +305 Use Proxy +400 Bad Request +401 Unauthorized +402 Payment Required +403 Forbidden +404 Not Found +405 Method Not Allowed +406 Not Acceptable +407 Proxy Authentication Required +408 Request Time-out +409 Conflict +410 Gone +411 Length Required +412 Precondition Failed +413 Request Entity Too Large +414 Request-URI Too Large +415 Unsupported Media Type +500 Internal Server Error +501 Not Implemented +502 Bad Gateway +503 Service Unavailable +504 Gateway Time-out +505 HTTP Version not supported |
