diff options
Diffstat (limited to 'lib/libfetch')
-rw-r--r-- | lib/libfetch/fetch.3 | 42 | ||||
-rw-r--r-- | lib/libfetch/fetch.c | 35 | ||||
-rw-r--r-- | lib/libfetch/fetch.h | 5 | ||||
-rw-r--r-- | lib/libfetch/file.c | 11 | ||||
-rw-r--r-- | lib/libfetch/ftp.c | 228 | ||||
-rw-r--r-- | lib/libfetch/http.c | 28 |
6 files changed, 236 insertions, 113 deletions
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3 index 0efb690..d29809e 100644 --- a/lib/libfetch/fetch.3 +++ b/lib/libfetch/fetch.3 @@ -31,22 +31,27 @@ .Nm fetchMakeURL , .Nm fetchParseURL , .Nm fetchFreeURL , +.Nm fetchXGetURL , .Nm fetchGetURL , .Nm fetchPutURL , .Nm fetchStatURL , .Nm fetchListURL , +.Nm fetchXGet , .Nm fetchGet , .Nm fetchPut , .Nm fetchStat , .Nm fetchList , +.Nm fetchXGetFile , .Nm fetchGetFile , .Nm fetchPutFile , .Nm fetchStatFile , .Nm fetchListFile , +.Nm fetchXGetHTTP , .Nm fetchGetHTTP , .Nm fetchPutHTTP , .Nm fetchStatHTTP , .Nm fetchListHTTP , +.Nm fetchXGetFTP , .Nm fetchGetFTP , .Nm fetchPutFTP , .Nm fetchStatFTP , @@ -65,6 +70,8 @@ .Ft void .Fn fetchFreeURL "struct url *URL" .Ft FILE * +.Fn fetchXGetURL "char *URL" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetURL "char *URL" "char *flags" .Ft FILE * .Fn fetchPutURL "char *URL" "char *flags" @@ -73,6 +80,8 @@ .Ft struct url_ent * .Fn fetchListURL "char *URL" "char *flags" .Ft FILE * +.Fn fetchXGet "struct url *URL" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGet "struct url *URL" "char *flags" .Ft FILE * .Fn fetchPut "struct url *URL" "char *flags" @@ -81,6 +90,8 @@ .Ft struct url_ent * .Fn fetchList "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetFile "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutFile "struct url *u" "char *flags" @@ -89,6 +100,8 @@ .Ft struct url_ent * .Fn fetchListFile "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetHTTP "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutHTTP "struct url *u" "char *flags" @@ -97,6 +110,8 @@ .Ft struct url_ent * .Fn fetchListHTTP "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetFTP "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutFTP "struct url *u" "char *flags" @@ -154,7 +169,8 @@ or should be freed using .Fn fetchFreeURL . .Pp -.Fn fetchGetURL +.Fn fetchXGetURL , +.Fn fetchGetURL , and .Fn fetchPutURL constitute the recommended interface to the @@ -163,6 +179,13 @@ library. They examine the URL passed to them to determine the transfer method, and call the appropriate lower-level functions to perform the actual transfer. +.Fn fetchXGetURL +also returns the remote document's metadata in the +.Fa url_stat +structure pointed to by the +.Fa us +argument. +.Pp The .Fa flags argument is a string of characters which specify transfer options. @@ -219,11 +242,13 @@ The pointer returned by should be freed using .Fn free . .Pp +.Fn fetchXGet , .Fn fetchGet , .Fn fetchPut and .Fn fetchStat are similar to +.Fn fetchXGetURL , .Fn fetchGetURL , .Fn fetchPutURL and @@ -234,6 +259,7 @@ a rather than a string. .Pp All of the +.Fn fetchXGetXXX , .Fn fetchGetXXX and .Fn fetchPutXXX @@ -242,11 +268,14 @@ write data from or to the requested document, respectively. Note that although the implementation details of the individual access methods vary, it can generally be assumed that a stream returned by one of the +.Fn fetchXGetXXX +or .Fn fetchGetXXX functions is read-only, and that a stream returned by one of the .Fn fetchPutXXX functions is write-only. .Sh FILE SCHEME +.Fn fetchXGetFile , .Fn fetchGetFile and .Fn fetchPutFile @@ -254,8 +283,10 @@ provide access to documents which are files in a locally mounted file system. Only the <document> component of the URL is used. .Pp +.Fn fetchXGetFile +and .Fn fetchGetFile -does not accept any flags. +do not accept any flags. .Pp .Fn fetchPutFile accepts the @@ -267,6 +298,7 @@ the stream returned by will be appended to the previous contents of the file, instead of replacing them. .Sh FTP SCHEME +.Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP @@ -286,6 +318,7 @@ port range (see If the .Fa d (direct) flag is specified, +.Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP @@ -297,6 +330,7 @@ library will attempt an anonymous login, with user name "ftp" and password "ftp". .Sh HTTP SCHEME The +.Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP @@ -307,6 +341,7 @@ even a chance that they comply with RFC2068. If the .Fa d (direct) flag is specified, +.Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP @@ -494,7 +529,8 @@ does not check that the result of an MDTM command is a valid date. .Pp The HTTP code needs a complete rewrite, or at least a serious cleanup. .Pp -The man page is poorly written and produces badly formatted text. +The man page is incomplete, poorly written and produces badly +formatted text. .Pp The error reporting mechanism is unsatisfactory. .Pp diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c index adb95c9..cb09efc 100644 --- a/lib/libfetch/fetch.c +++ b/lib/libfetch/fetch.c @@ -67,22 +67,23 @@ static struct fetcherr _url_errlist[] = { /* * Select the appropriate protocol for the URL scheme, and return a * read-only stream connected to the document referenced by the URL. + * Also fill out the struct url_stat. */ FILE * -fetchGet(struct url *URL, char *flags) +fetchXGet(struct url *URL, struct url_stat *us, char *flags) { int direct; direct = (flags && strchr(flags, 'd')); if (strcasecmp(URL->scheme, "file") == 0) - return fetchGetFile(URL, flags); + return fetchXGetFile(URL, us, flags); else if (strcasecmp(URL->scheme, "http") == 0) - return fetchGetHTTP(URL, flags); + return fetchXGetHTTP(URL, us, flags); else if (strcasecmp(URL->scheme, "ftp") == 0) { if (!direct && getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL) - return fetchGetHTTP(URL, flags); - return fetchGetFTP(URL, flags); + return fetchXGetHTTP(URL, us, flags); + return fetchXGetFTP(URL, us, flags); } else { _url_seterr(URL_BAD_SCHEME); return NULL; @@ -91,6 +92,16 @@ fetchGet(struct url *URL, char *flags) /* * Select the appropriate protocol for the URL scheme, and return a + * read-only stream connected to the document referenced by the URL. + */ +FILE * +fetchGet(struct url *URL, char *flags) +{ + return fetchXGet(URL, NULL, flags); +} + +/* + * Select the appropriate protocol for the URL scheme, and return a * write-only stream connected to the document referenced by the URL. */ FILE * @@ -165,10 +176,10 @@ fetchList(struct url *URL, char *flags) } /* - * Attempt to parse the given URL; if successful, call fetchGet(). + * Attempt to parse the given URL; if successful, call fetchXGet(). */ FILE * -fetchGetURL(char *URL, char *flags) +fetchXGetURL(char *URL, struct url_stat *us, char *flags) { struct url *u; FILE *f; @@ -176,12 +187,20 @@ fetchGetURL(char *URL, char *flags) if ((u = fetchParseURL(URL)) == NULL) return NULL; - f = fetchGet(u, flags); + f = fetchXGet(u, us, flags); fetchFreeURL(u); return f; } +/* + * Attempt to parse the given URL; if successful, call fetchGet(). + */ +FILE * +fetchGetURL(char *URL, char *flags) +{ + return fetchXGetURL(URL, NULL, flags); +} /* * Attempt to parse the given URL; if successful, call fetchPut(). diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h index c503dac..9d7fa9b 100644 --- a/lib/libfetch/fetch.h +++ b/lib/libfetch/fetch.h @@ -81,28 +81,33 @@ struct url_ent { #define FETCH_VERBOSE 19 /* FILE-specific functions */ +FILE *fetchXGetFile(struct url *, struct url_stat *, char *); FILE *fetchGetFile(struct url *, char *); FILE *fetchPutFile(struct url *, char *); int fetchStatFile(struct url *, struct url_stat *, char *); struct url_ent *fetchListFile(struct url *, char *); /* HTTP-specific functions */ +FILE *fetchXGetHTTP(struct url *, struct url_stat *, char *); FILE *fetchGetHTTP(struct url *, char *); FILE *fetchPutHTTP(struct url *, char *); int fetchStatHTTP(struct url *, struct url_stat *, char *); struct url_ent *fetchListHTTP(struct url *, char *); /* FTP-specific functions */ +FILE *fetchXGetFTP(struct url *, struct url_stat *, char *); FILE *fetchGetFTP(struct url *, char *); FILE *fetchPutFTP(struct url *, char *); int fetchStatFTP(struct url *, struct url_stat *, char *); struct url_ent *fetchListFTP(struct url *, char *); /* Generic functions */ +FILE *fetchXGetURL(char *, struct url_stat *, char *); FILE *fetchGetURL(char *, char *); FILE *fetchPutURL(char *, char *); int fetchStatURL(char *, struct url_stat *, char *); struct url_ent *fetchListURL(char *, char *); +FILE *fetchXGet(struct url *, struct url_stat *, char *); FILE *fetchGet(struct url *, char *); FILE *fetchPut(struct url *, char *); int fetchStat(struct url *, struct url_stat *, char *); diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c index 0da44c9..7cf6efd 100644 --- a/lib/libfetch/file.c +++ b/lib/libfetch/file.c @@ -39,10 +39,13 @@ #include "common.h" FILE * -fetchGetFile(struct url *u, char *flags) +fetchXGetFile(struct url *u, struct url_stat *us, char *flags) { FILE *f; + if (us && fetchStatFile(u, us, flags) == -1) + return NULL; + f = fopen(u->doc, "r"); if (f == NULL) @@ -57,6 +60,12 @@ fetchGetFile(struct url *u, char *flags) } FILE * +fetchGetFile(struct url *u, char *flags) +{ + return fetchXGetFile(u, NULL, flags); +} + +FILE * fetchPutFile(struct url *u, char *flags) { FILE *f; diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c index 20fc50e..b141bc1 100644 --- a/lib/libfetch/ftp.c +++ b/lib/libfetch/ftp.c @@ -194,6 +194,111 @@ _ftp_cmd(int cd, char *fmt, ...) } /* + * Return a pointer to the filename part of a path + */ +static char * +_ftp_filename(char *file) +{ + char *s; + + if ((s = strrchr(file, '/')) == NULL) + return file; + else + return s + 1; +} + +/* + * Change working directory to the directory that contains the + * specified file. + */ +static int +_ftp_cwd(int cd, char *file) +{ + char *s; + int e; + + if ((s = strrchr(file, '/')) == NULL) { + e = _ftp_cmd(cd, "CWD /"); + } else { + e = _ftp_cmd(cd, "CWD %.*s", s - file, file); + } + if (e != FTP_FILE_ACTION_OK) { + _ftp_seterr(e); + return -1; + } + return 0; +} + +/* + * Request and parse file stats + */ +static int +_ftp_stat(int cd, char *file, struct url_stat *us) +{ + char *ln, *s; + struct tm tm; + time_t t; + int e; + + if ((s = strrchr(file, '/')) == NULL) + s = file; + else + ++s; + + if ((e = _ftp_cmd(cd, "SIZE %s", s)) != FTP_FILE_STATUS) { + _ftp_seterr(e); + return -1; + } + for (ln = last_reply + 4; *ln && isspace(*ln); ln++) + /* nothing */ ; + for (us->size = 0; *ln && isdigit(*ln); ln++) + us->size = us->size * 10 + *ln - '0'; + if (*ln && !isspace(*ln)) { + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + DEBUG(fprintf(stderr, "size: [\033[1m%lld\033[m]\n", us->size)); + + if ((e = _ftp_cmd(cd, "MDTM %s", s)) != FTP_FILE_STATUS) { + _ftp_seterr(e); + return -1; + } + for (ln = last_reply + 4; *ln && isspace(*ln); ln++) + /* nothing */ ; + switch (strspn(ln, "0123456789")) { + case 14: + break; + case 15: + ln++; + ln[0] = '2'; + ln[1] = '0'; + break; + default: + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + if (sscanf(ln, "%04d%02d%02d%02d%02d%02d", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) { + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + tm.tm_mon--; + tm.tm_year -= 1900; + tm.tm_isdst = -1; + t = timegm(&tm); + if (t == (time_t)-1) + t = time(NULL); + us->mtime = t; + us->atime = t; + DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " + "%02d:%02d:%02d\033[m]\n", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec)); + return 0; +} + +/* * Transfer file */ static FILE * @@ -218,30 +323,6 @@ _ftp_transfer(int cd, char *oper, char *file, if (!pasv && (s = getenv("FTP_PASSIVE_MODE")) != NULL) pasv = (strncasecmp(s, "no", 2) != 0); - /* change directory */ - if (((s = strrchr(file, '/')) != NULL) && (s != file)) { - *s = 0; - if (verbose) - _fetch_info("changing directory to %s", file); - if ((e = _ftp_cmd(cd, "CWD %s", file)) != FTP_FILE_ACTION_OK) { - *s = '/'; - if (e != -1) - _ftp_seterr(e); - return NULL; - } - *s++ = '/'; - } else { - if (verbose) - _fetch_info("changing directory to /"); - if ((e = _ftp_cmd(cd, "CWD /")) != FTP_FILE_ACTION_OK) { - if (e != -1) - _ftp_seterr(e); - return NULL; - } - } - - /* s now points to file name */ - /* find our own address, bind, and listen */ l = sizeof sin; if (getsockname(cd, (struct sockaddr *)&sin, &l) == -1) @@ -366,7 +447,7 @@ _ftp_transfer(int cd, char *oper, char *file, /* make the server initiate the transfer */ if (verbose) _fetch_info("initiating transfer"); - e = _ftp_cmd(cd, "%s %s", oper, s); + e = _ftp_cmd(cd, "%s %s", oper, _ftp_filename(file)); if (e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -456,7 +537,7 @@ _ftp_transfer(int cd, char *oper, char *file, /* make the server initiate the transfer */ if (verbose) _fetch_info("initiating transfer"); - e = _ftp_cmd(cd, "%s %s", oper, s); + e = _ftp_cmd(cd, "%s %s", oper, _ftp_filename(file)); if (e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -675,10 +756,10 @@ _ftp_cached_connect(struct url *url, char *flags) } /* - * Get file + * Get and stat file */ FILE * -fetchGetFTP(struct url *url, char *flags) +fetchXGetFTP(struct url *url, struct url_stat *us, char *flags) { int cd; @@ -686,11 +767,28 @@ fetchGetFTP(struct url *url, char *flags) if ((cd = _ftp_cached_connect(url, flags)) == NULL) return NULL; + /* change directory */ + if (_ftp_cwd(cd, url->doc) == -1) + return NULL; + + /* stat file */ + if (us && _ftp_stat(cd, url->doc, us) == -1) + return NULL; + /* initiate the transfer */ return _ftp_transfer(cd, "RETR", url->doc, "r", url->offset, flags); } /* + * Get file + */ +FILE * +fetchGetFTP(struct url *url, char *flags) +{ + return fetchXGetFTP(url, NULL, flags); +} + +/* * Put file */ FILE * @@ -702,6 +800,10 @@ fetchPutFTP(struct url *url, char *flags) if ((cd = _ftp_cached_connect(url, flags)) == NULL) return NULL; + /* change directory */ + if (_ftp_cwd(cd, url->doc) == -1) + return NULL; + /* initiate the transfer */ return _ftp_transfer(cd, (flags && strchr(flags, 'a')) ? "APPE" : "STOR", url->doc, "w", url->offset, flags); @@ -713,10 +815,7 @@ fetchPutFTP(struct url *url, char *flags) int fetchStatFTP(struct url *url, struct url_stat *us, char *flags) { - char *ln, *s; - struct tm tm; - time_t t; - int e, cd; + int cd; us->size = -1; us->atime = us->mtime = 0; @@ -726,70 +825,11 @@ fetchStatFTP(struct url *url, struct url_stat *us, char *flags) return -1; /* change directory */ - if (((s = strrchr(url->doc, '/')) != NULL) && (s != url->doc)) { - *s = 0; - if ((e = _ftp_cmd(cd, "CWD %s", url->doc)) != FTP_FILE_ACTION_OK) { - *s = '/'; - goto ouch; - } - *s++ = '/'; - } else { - if ((e = _ftp_cmd(cd, "CWD /")) != FTP_FILE_ACTION_OK) - goto ouch; - } - - /* s now points to file name */ - - if (_ftp_cmd(cd, "SIZE %s", s) != FTP_FILE_STATUS) - goto ouch; - for (ln = last_reply + 4; *ln && isspace(*ln); ln++) - /* nothing */ ; - for (us->size = 0; *ln && isdigit(*ln); ln++) - us->size = us->size * 10 + *ln - '0'; - if (*ln && !isspace(*ln)) { - _ftp_seterr(FTP_PROTOCOL_ERROR); + if (_ftp_cwd(cd, url->doc) == -1) return -1; - } - DEBUG(fprintf(stderr, "size: [\033[1m%lld\033[m]\n", us->size)); - if ((e = _ftp_cmd(cd, "MDTM %s", s)) != FTP_FILE_STATUS) - goto ouch; - for (ln = last_reply + 4; *ln && isspace(*ln); ln++) - /* nothing */ ; - e = FTP_PROTOCOL_ERROR; - switch (strspn(ln, "0123456789")) { - case 14: - break; - case 15: - ln++; - ln[0] = '2'; - ln[1] = '0'; - break; - default: - goto ouch; - } - if (sscanf(ln, "%04d%02d%02d%02d%02d%02d", - &tm.tm_year, &tm.tm_mon, &tm.tm_mday, - &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) - goto ouch; - tm.tm_mon--; - tm.tm_year -= 1900; - tm.tm_isdst = -1; - t = timegm(&tm); - if (t == (time_t)-1) - t = time(NULL); - us->mtime = t; - us->atime = t; - DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " - "%02d:%02d:%02d\033[m]\n", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec)); - return 0; - -ouch: - if (e != -1) - _ftp_seterr(e); - return -1; + /* stat file */ + return _ftp_stat(cd, url->doc, us); } /* diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index a384e04..ce748c5 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -772,8 +772,10 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) for (url = URL, i = 0; i < n; ++i) { new = NULL; - us->size = -1; - us->atime = us->mtime = 0; + if (us) { + us->size = -1; + us->atime = us->mtime = 0; + } chunked = 0; need_auth = 0; offset = 0; @@ -883,13 +885,15 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) _http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; case hdr_content_length: - us->size = _http_parse_length(p); + if (us) + us->size = _http_parse_length(p); break; case hdr_content_range: offset = _http_parse_range(p); break; case hdr_last_modified: - us->atime = us->mtime = _http_parse_mtime(p); + if (us) + us->atime = us->mtime = _http_parse_mtime(p); break; case hdr_location: if (!HTTP_REDIRECT(code)) @@ -978,16 +982,26 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) */ /* + * Retrieve and stat a file by HTTP + */ +FILE * +fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags) +{ + return _http_request(URL, "GET", us, flags); +} + +/* * Retrieve a file by HTTP */ FILE * fetchGetHTTP(struct url *URL, char *flags) { - struct url_stat us; - - return _http_request(URL, "GET", &us, flags); + return fetchXGetHTTP(URL, NULL, flags); } +/* + * Store a file by HTTP + */ FILE * fetchPutHTTP(struct url *URL, char *flags) { |