diff options
| -rw-r--r-- | lib/libfetch/common.c | 4 | ||||
| -rw-r--r-- | lib/libfetch/fetch.3 | 42 | ||||
| -rw-r--r-- | lib/libfetch/fetch.c | 36 | ||||
| -rw-r--r-- | lib/libfetch/fetch.h | 7 | ||||
| -rw-r--r-- | lib/libfetch/file.c | 11 | ||||
| -rw-r--r-- | lib/libfetch/ftp.c | 245 | ||||
| -rw-r--r-- | lib/libfetch/http.c | 30 | ||||
| -rw-r--r-- | usr.bin/fetch/fetch.1 | 10 | ||||
| -rw-r--r-- | usr.bin/fetch/fetch.c | 119 |
9 files changed, 344 insertions, 160 deletions
diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index e4dbf3aa60cf..c03b1f52b0ff 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -262,7 +262,7 @@ _fetch_getln(int fd, char **buf, size_t *size, size_t *len) } r = select(fd+1, &readfds, NULL, NULL, &wait); if (r == -1) { - if (errno == EINTR) + if (errno == EINTR && fetchRestartCalls) continue; /* EBADF or EINVAL: shouldn't happen */ return -1; @@ -274,7 +274,7 @@ _fetch_getln(int fd, char **buf, size_t *size, size_t *len) if (r == 0) break; if (r == -1) { - if (errno == EINTR) + if (errno == EINTR && fetchRestartCalls) continue; /* any other error is bad news */ return -1; diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3 index 0efb6904e20f..d29809e69248 100644 --- a/lib/libfetch/fetch.3 +++ b/lib/libfetch/fetch.3 @@ -31,22 +31,27 @@ .Nm fetchMakeURL , .Nm fetchParseURL , .Nm fetchFreeURL , +.Nm fetchXGetURL , .Nm fetchGetURL , .Nm fetchPutURL , .Nm fetchStatURL , .Nm fetchListURL , +.Nm fetchXGet , .Nm fetchGet , .Nm fetchPut , .Nm fetchStat , .Nm fetchList , +.Nm fetchXGetFile , .Nm fetchGetFile , .Nm fetchPutFile , .Nm fetchStatFile , .Nm fetchListFile , +.Nm fetchXGetHTTP , .Nm fetchGetHTTP , .Nm fetchPutHTTP , .Nm fetchStatHTTP , .Nm fetchListHTTP , +.Nm fetchXGetFTP , .Nm fetchGetFTP , .Nm fetchPutFTP , .Nm fetchStatFTP , @@ -65,6 +70,8 @@ .Ft void .Fn fetchFreeURL "struct url *URL" .Ft FILE * +.Fn fetchXGetURL "char *URL" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetURL "char *URL" "char *flags" .Ft FILE * .Fn fetchPutURL "char *URL" "char *flags" @@ -73,6 +80,8 @@ .Ft struct url_ent * .Fn fetchListURL "char *URL" "char *flags" .Ft FILE * +.Fn fetchXGet "struct url *URL" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGet "struct url *URL" "char *flags" .Ft FILE * .Fn fetchPut "struct url *URL" "char *flags" @@ -81,6 +90,8 @@ .Ft struct url_ent * .Fn fetchList "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetFile "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutFile "struct url *u" "char *flags" @@ -89,6 +100,8 @@ .Ft struct url_ent * .Fn fetchListFile "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetHTTP "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutHTTP "struct url *u" "char *flags" @@ -97,6 +110,8 @@ .Ft struct url_ent * .Fn fetchListHTTP "struct url *" "char *flags" .Ft FILE * +.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "char *flags" +.Ft FILE * .Fn fetchGetFTP "struct url *u" "char *flags" .Ft FILE * .Fn fetchPutFTP "struct url *u" "char *flags" @@ -154,7 +169,8 @@ or should be freed using .Fn fetchFreeURL . .Pp -.Fn fetchGetURL +.Fn fetchXGetURL , +.Fn fetchGetURL , and .Fn fetchPutURL constitute the recommended interface to the @@ -163,6 +179,13 @@ library. They examine the URL passed to them to determine the transfer method, and call the appropriate lower-level functions to perform the actual transfer. +.Fn fetchXGetURL +also returns the remote document's metadata in the +.Fa url_stat +structure pointed to by the +.Fa us +argument. +.Pp The .Fa flags argument is a string of characters which specify transfer options. @@ -219,11 +242,13 @@ The pointer returned by should be freed using .Fn free . .Pp +.Fn fetchXGet , .Fn fetchGet , .Fn fetchPut and .Fn fetchStat are similar to +.Fn fetchXGetURL , .Fn fetchGetURL , .Fn fetchPutURL and @@ -234,6 +259,7 @@ a rather than a string. .Pp All of the +.Fn fetchXGetXXX , .Fn fetchGetXXX and .Fn fetchPutXXX @@ -242,11 +268,14 @@ write data from or to the requested document, respectively. Note that although the implementation details of the individual access methods vary, it can generally be assumed that a stream returned by one of the +.Fn fetchXGetXXX +or .Fn fetchGetXXX functions is read-only, and that a stream returned by one of the .Fn fetchPutXXX functions is write-only. .Sh FILE SCHEME +.Fn fetchXGetFile , .Fn fetchGetFile and .Fn fetchPutFile @@ -254,8 +283,10 @@ provide access to documents which are files in a locally mounted file system. Only the <document> component of the URL is used. .Pp +.Fn fetchXGetFile +and .Fn fetchGetFile -does not accept any flags. +do not accept any flags. .Pp .Fn fetchPutFile accepts the @@ -267,6 +298,7 @@ the stream returned by will be appended to the previous contents of the file, instead of replacing them. .Sh FTP SCHEME +.Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP @@ -286,6 +318,7 @@ port range (see If the .Fa d (direct) flag is specified, +.Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP @@ -297,6 +330,7 @@ library will attempt an anonymous login, with user name "ftp" and password "ftp". .Sh HTTP SCHEME The +.Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP @@ -307,6 +341,7 @@ even a chance that they comply with RFC2068. If the .Fa d (direct) flag is specified, +.Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP @@ -494,7 +529,8 @@ does not check that the result of an MDTM command is a valid date. .Pp The HTTP code needs a complete rewrite, or at least a serious cleanup. .Pp -The man page is poorly written and produces badly formatted text. +The man page is incomplete, poorly written and produces badly +formatted text. .Pp The error reporting mechanism is unsatisfactory. .Pp diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c index 5d83db5b5de8..cb09efcfeac4 100644 --- a/lib/libfetch/fetch.c +++ b/lib/libfetch/fetch.c @@ -43,6 +43,7 @@ int fetchLastErrCode; char fetchLastErrString[MAXERRSTRING]; int fetchTimeout; +int fetchRestartCalls = 1; /*** Local data **************************************************************/ @@ -66,22 +67,23 @@ static struct fetcherr _url_errlist[] = { /* * Select the appropriate protocol for the URL scheme, and return a * read-only stream connected to the document referenced by the URL. + * Also fill out the struct url_stat. */ FILE * -fetchGet(struct url *URL, char *flags) +fetchXGet(struct url *URL, struct url_stat *us, char *flags) { int direct; direct = (flags && strchr(flags, 'd')); if (strcasecmp(URL->scheme, "file") == 0) - return fetchGetFile(URL, flags); + return fetchXGetFile(URL, us, flags); else if (strcasecmp(URL->scheme, "http") == 0) - return fetchGetHTTP(URL, flags); + return fetchXGetHTTP(URL, us, flags); else if (strcasecmp(URL->scheme, "ftp") == 0) { if (!direct && getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL) - return fetchGetHTTP(URL, flags); - return fetchGetFTP(URL, flags); + return fetchXGetHTTP(URL, us, flags); + return fetchXGetFTP(URL, us, flags); } else { _url_seterr(URL_BAD_SCHEME); return NULL; @@ -90,6 +92,16 @@ fetchGet(struct url *URL, char *flags) /* * Select the appropriate protocol for the URL scheme, and return a + * read-only stream connected to the document referenced by the URL. + */ +FILE * +fetchGet(struct url *URL, char *flags) +{ + return fetchXGet(URL, NULL, flags); +} + +/* + * Select the appropriate protocol for the URL scheme, and return a * write-only stream connected to the document referenced by the URL. */ FILE * @@ -164,10 +176,10 @@ fetchList(struct url *URL, char *flags) } /* - * Attempt to parse the given URL; if successful, call fetchGet(). + * Attempt to parse the given URL; if successful, call fetchXGet(). */ FILE * -fetchGetURL(char *URL, char *flags) +fetchXGetURL(char *URL, struct url_stat *us, char *flags) { struct url *u; FILE *f; @@ -175,12 +187,20 @@ fetchGetURL(char *URL, char *flags) if ((u = fetchParseURL(URL)) == NULL) return NULL; - f = fetchGet(u, flags); + f = fetchXGet(u, us, flags); fetchFreeURL(u); return f; } +/* + * Attempt to parse the given URL; if successful, call fetchGet(). + */ +FILE * +fetchGetURL(char *URL, char *flags) +{ + return fetchXGetURL(URL, NULL, flags); +} /* * Attempt to parse the given URL; if successful, call fetchPut(). diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h index 2395220af8ef..9d7fa9ba1a2b 100644 --- a/lib/libfetch/fetch.h +++ b/lib/libfetch/fetch.h @@ -81,29 +81,33 @@ struct url_ent { #define FETCH_VERBOSE 19 /* FILE-specific functions */ +FILE *fetchXGetFile(struct url *, struct url_stat *, char *); FILE *fetchGetFile(struct url *, char *); FILE *fetchPutFile(struct url *, char *); int fetchStatFile(struct url *, struct url_stat *, char *); struct url_ent *fetchListFile(struct url *, char *); /* HTTP-specific functions */ -char *fetchContentType(FILE *); +FILE *fetchXGetHTTP(struct url *, struct url_stat *, char *); FILE *fetchGetHTTP(struct url *, char *); FILE *fetchPutHTTP(struct url *, char *); int fetchStatHTTP(struct url *, struct url_stat *, char *); struct url_ent *fetchListHTTP(struct url *, char *); /* FTP-specific functions */ +FILE *fetchXGetFTP(struct url *, struct url_stat *, char *); FILE *fetchGetFTP(struct url *, char *); FILE *fetchPutFTP(struct url *, char *); int fetchStatFTP(struct url *, struct url_stat *, char *); struct url_ent *fetchListFTP(struct url *, char *); /* Generic functions */ +FILE *fetchXGetURL(char *, struct url_stat *, char *); FILE *fetchGetURL(char *, char *); FILE *fetchPutURL(char *, char *); int fetchStatURL(char *, struct url_stat *, char *); struct url_ent *fetchListURL(char *, char *); +FILE *fetchXGet(struct url *, struct url_stat *, char *); FILE *fetchGet(struct url *, char *); FILE *fetchPut(struct url *, char *); int fetchStat(struct url *, struct url_stat *, char *); @@ -119,5 +123,6 @@ extern int fetchLastErrCode; #define MAXERRSTRING 256 extern char fetchLastErrString[MAXERRSTRING]; extern int fetchTimeout; +extern int fetchRestartCalls; #endif diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c index 0da44c9d5fa3..7cf6efdb6d0e 100644 --- a/lib/libfetch/file.c +++ b/lib/libfetch/file.c @@ -39,10 +39,13 @@ #include "common.h" FILE * -fetchGetFile(struct url *u, char *flags) +fetchXGetFile(struct url *u, struct url_stat *us, char *flags) { FILE *f; + if (us && fetchStatFile(u, us, flags) == -1) + return NULL; + f = fopen(u->doc, "r"); if (f == NULL) @@ -57,6 +60,12 @@ fetchGetFile(struct url *u, char *flags) } FILE * +fetchGetFile(struct url *u, char *flags) +{ + return fetchXGetFile(u, NULL, flags); +} + +FILE * fetchPutFile(struct url *u, char *flags) { FILE *f; diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c index 60ffe96669b9..b141bc173f74 100644 --- a/lib/libfetch/ftp.c +++ b/lib/libfetch/ftp.c @@ -90,6 +90,7 @@ #define FTP_NEED_ACCOUNT 332 #define FTP_FILE_OK 350 #define FTP_SYNTAX_ERROR 500 +#define FTP_PROTOCOL_ERROR 999 static struct url cached_host; static int cached_socket; @@ -149,7 +150,7 @@ _ftp_chkerr(int cd) last_reply[lr_length] = 0; if (!isftpreply(last_reply)) { - _ftp_seterr(999); + _ftp_seterr(FTP_PROTOCOL_ERROR); return -1; } @@ -193,6 +194,111 @@ _ftp_cmd(int cd, char *fmt, ...) } /* + * Return a pointer to the filename part of a path + */ +static char * +_ftp_filename(char *file) +{ + char *s; + + if ((s = strrchr(file, '/')) == NULL) + return file; + else + return s + 1; +} + +/* + * Change working directory to the directory that contains the + * specified file. + */ +static int +_ftp_cwd(int cd, char *file) +{ + char *s; + int e; + + if ((s = strrchr(file, '/')) == NULL) { + e = _ftp_cmd(cd, "CWD /"); + } else { + e = _ftp_cmd(cd, "CWD %.*s", s - file, file); + } + if (e != FTP_FILE_ACTION_OK) { + _ftp_seterr(e); + return -1; + } + return 0; +} + +/* + * Request and parse file stats + */ +static int +_ftp_stat(int cd, char *file, struct url_stat *us) +{ + char *ln, *s; + struct tm tm; + time_t t; + int e; + + if ((s = strrchr(file, '/')) == NULL) + s = file; + else + ++s; + + if ((e = _ftp_cmd(cd, "SIZE %s", s)) != FTP_FILE_STATUS) { + _ftp_seterr(e); + return -1; + } + for (ln = last_reply + 4; *ln && isspace(*ln); ln++) + /* nothing */ ; + for (us->size = 0; *ln && isdigit(*ln); ln++) + us->size = us->size * 10 + *ln - '0'; + if (*ln && !isspace(*ln)) { + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + DEBUG(fprintf(stderr, "size: [\033[1m%lld\033[m]\n", us->size)); + + if ((e = _ftp_cmd(cd, "MDTM %s", s)) != FTP_FILE_STATUS) { + _ftp_seterr(e); + return -1; + } + for (ln = last_reply + 4; *ln && isspace(*ln); ln++) + /* nothing */ ; + switch (strspn(ln, "0123456789")) { + case 14: + break; + case 15: + ln++; + ln[0] = '2'; + ln[1] = '0'; + break; + default: + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + if (sscanf(ln, "%04d%02d%02d%02d%02d%02d", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) { + _ftp_seterr(FTP_PROTOCOL_ERROR); + return -1; + } + tm.tm_mon--; + tm.tm_year -= 1900; + tm.tm_isdst = -1; + t = timegm(&tm); + if (t == (time_t)-1) + t = time(NULL); + us->mtime = t; + us->atime = t; + DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " + "%02d:%02d:%02d\033[m]\n", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec)); + return 0; +} + +/* * Transfer file */ static FILE * @@ -217,30 +323,6 @@ _ftp_transfer(int cd, char *oper, char *file, if (!pasv && (s = getenv("FTP_PASSIVE_MODE")) != NULL) pasv = (strncasecmp(s, "no", 2) != 0); - /* change directory */ - if (((s = strrchr(file, '/')) != NULL) && (s != file)) { - *s = 0; - if (verbose) - _fetch_info("changing directory to %s", file); - if ((e = _ftp_cmd(cd, "CWD %s", file)) != FTP_FILE_ACTION_OK) { - *s = '/'; - if (e != -1) - _ftp_seterr(e); - return NULL; - } - *s++ = '/'; - } else { - if (verbose) - _fetch_info("changing directory to /"); - if ((e = _ftp_cmd(cd, "CWD /")) != FTP_FILE_ACTION_OK) { - if (e != -1) - _ftp_seterr(e); - return NULL; - } - } - - /* s now points to file name */ - /* find our own address, bind, and listen */ l = sizeof sin; if (getsockname(cd, (struct sockaddr *)&sin, &l) == -1) @@ -277,7 +359,7 @@ _ftp_transfer(int cd, char *oper, char *file, } break; default: - e = 999; /* XXX: error code should be prepared */ + e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */ goto ouch; } @@ -292,14 +374,14 @@ _ftp_transfer(int cd, char *oper, char *file, for (p = ln + 3; *p && !isdigit(*p); p++) /* nothing */ ; if (!*p) { - e = 999; + e = FTP_PROTOCOL_ERROR; goto ouch; } l = (e == FTP_PASSIVE_MODE ? 6 : 21); for (i = 0; *p && i < l; i++, p++) addr[i] = strtol(p, &p, 10); if (i < l) { - e = 999; + e = FTP_PROTOCOL_ERROR; goto ouch; } break; @@ -307,7 +389,7 @@ _ftp_transfer(int cd, char *oper, char *file, for (p = ln + 3; *p && *p != '('; p++) /* nothing */ ; if (!*p) { - e = 999; + e = FTP_PROTOCOL_ERROR; goto ouch; } ++p; @@ -315,7 +397,7 @@ _ftp_transfer(int cd, char *oper, char *file, &port, &addr[3]) != 5 || addr[0] != addr[1] || addr[0] != addr[2] || addr[0] != addr[3]) { - e = 999; + e = FTP_PROTOCOL_ERROR; goto ouch; } break; @@ -352,7 +434,7 @@ _ftp_transfer(int cd, char *oper, char *file, } break; default: - e = 999; /* XXX: error code should be prepared */ + e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */ break; } @@ -365,7 +447,7 @@ _ftp_transfer(int cd, char *oper, char *file, /* make the server initiate the transfer */ if (verbose) _fetch_info("initiating transfer"); - e = _ftp_cmd(cd, "%s %s", oper, s); + e = _ftp_cmd(cd, "%s %s", oper, _ftp_filename(file)); if (e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -441,7 +523,7 @@ _ftp_transfer(int cd, char *oper, char *file, } break; default: - e = 999; /* XXX: error code should be prepared */ + e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */ goto ouch; } if (e != FTP_OK) @@ -455,7 +537,7 @@ _ftp_transfer(int cd, char *oper, char *file, /* make the server initiate the transfer */ if (verbose) _fetch_info("initiating transfer"); - e = _ftp_cmd(cd, "%s %s", oper, s); + e = _ftp_cmd(cd, "%s %s", oper, _ftp_filename(file)); if (e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -674,10 +756,10 @@ _ftp_cached_connect(struct url *url, char *flags) } /* - * Get file + * Get and stat file */ FILE * -fetchGetFTP(struct url *url, char *flags) +fetchXGetFTP(struct url *url, struct url_stat *us, char *flags) { int cd; @@ -685,11 +767,28 @@ fetchGetFTP(struct url *url, char *flags) if ((cd = _ftp_cached_connect(url, flags)) == NULL) return NULL; + /* change directory */ + if (_ftp_cwd(cd, url->doc) == -1) + return NULL; + + /* stat file */ + if (us && _ftp_stat(cd, url->doc, us) == -1) + return NULL; + /* initiate the transfer */ return _ftp_transfer(cd, "RETR", url->doc, "r", url->offset, flags); } /* + * Get file + */ +FILE * +fetchGetFTP(struct url *url, char *flags) +{ + return fetchXGetFTP(url, NULL, flags); +} + +/* * Put file */ FILE * @@ -701,6 +800,10 @@ fetchPutFTP(struct url *url, char *flags) if ((cd = _ftp_cached_connect(url, flags)) == NULL) return NULL; + /* change directory */ + if (_ftp_cwd(cd, url->doc) == -1) + return NULL; + /* initiate the transfer */ return _ftp_transfer(cd, (flags && strchr(flags, 'a')) ? "APPE" : "STOR", url->doc, "w", url->offset, flags); @@ -712,10 +815,7 @@ fetchPutFTP(struct url *url, char *flags) int fetchStatFTP(struct url *url, struct url_stat *us, char *flags) { - char *ln, *s; - struct tm tm; - time_t t; - int e, cd; + int cd; us->size = -1; us->atime = us->mtime = 0; @@ -725,70 +825,11 @@ fetchStatFTP(struct url *url, struct url_stat *us, char *flags) return -1; /* change directory */ - if (((s = strrchr(url->doc, '/')) != NULL) && (s != url->doc)) { - *s = 0; - if ((e = _ftp_cmd(cd, "CWD %s", url->doc)) != FTP_FILE_ACTION_OK) { - *s = '/'; - goto ouch; - } - *s++ = '/'; - } else { - if ((e = _ftp_cmd(cd, "CWD /")) != FTP_FILE_ACTION_OK) - goto ouch; - } - - /* s now points to file name */ - - if (_ftp_cmd(cd, "SIZE %s", s) != FTP_FILE_STATUS) - goto ouch; - for (ln = last_reply + 4; *ln && isspace(*ln); ln++) - /* nothing */ ; - for (us->size = 0; *ln && isdigit(*ln); ln++) - us->size = us->size * 10 + *ln - '0'; - if (*ln && !isspace(*ln)) { - _ftp_seterr(999); + if (_ftp_cwd(cd, url->doc) == -1) return -1; - } - DEBUG(fprintf(stderr, "size: [\033[1m%lld\033[m]\n", us->size)); - - if ((e = _ftp_cmd(cd, "MDTM %s", s)) != FTP_FILE_STATUS) - goto ouch; - for (ln = last_reply + 4; *ln && isspace(*ln); ln++) - /* nothing */ ; - e = 999; - switch (strspn(ln, "0123456789")) { - case 14: - break; - case 15: - ln++; - ln[0] = '2'; - ln[1] = '0'; - break; - default: - goto ouch; - } - if (sscanf(ln, "%04d%02d%02d%02d%02d%02d", - &tm.tm_year, &tm.tm_mon, &tm.tm_mday, - &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) - goto ouch; - tm.tm_mon--; - tm.tm_year -= 1900; - tm.tm_isdst = -1; - t = timegm(&tm); - if (t == (time_t)-1) - t = time(NULL); - us->mtime = t; - us->atime = t; - DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " - "%02d:%02d:%02d\033[m]\n", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec)); - return 0; -ouch: - if (e != -1) - _ftp_seterr(e); - return -1; + /* stat file */ + return _ftp_stat(cd, url->doc, us); } /* diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index c9b2802b2e66..ce748c5c646f 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -772,12 +772,13 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) for (url = URL, i = 0; i < n; ++i) { new = NULL; - us->size = -1; - us->atime = us->mtime = 0; + if (us) { + us->size = -1; + us->atime = us->mtime = 0; + } chunked = 0; need_auth = 0; offset = 0; - fd = -1; retry: /* connect to server or proxy */ if ((fd = _http_connect(url, &proxy, flags)) == -1) @@ -884,13 +885,15 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) _http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; case hdr_content_length: - us->size = _http_parse_length(p); + if (us) + us->size = _http_parse_length(p); break; case hdr_content_range: offset = _http_parse_range(p); break; case hdr_last_modified: - us->atime = us->mtime = _http_parse_mtime(p); + if (us) + us->atime = us->mtime = _http_parse_mtime(p); break; case hdr_location: if (!HTTP_REDIRECT(code)) @@ -935,6 +938,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) /* we have a redirect */ close(fd); + fd = -1; if (url != URL) fetchFreeURL(url); url = new; @@ -978,16 +982,26 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags) */ /* + * Retrieve and stat a file by HTTP + */ +FILE * +fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags) +{ + return _http_request(URL, "GET", us, flags); +} + +/* * Retrieve a file by HTTP */ FILE * fetchGetHTTP(struct url *URL, char *flags) { - struct url_stat us; - - return _http_request(URL, "GET", &us, flags); + return fetchXGetHTTP(URL, NULL, flags); } +/* + * Store a file by HTTP + */ FILE * fetchPutHTTP(struct url *URL, char *flags) { diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1 index 186ba773b8cd..da0aec3ecdc7 100644 --- a/usr.bin/fetch/fetch.1 +++ b/usr.bin/fetch/fetch.1 @@ -118,6 +118,11 @@ rather than trying to copy it. .It Fl m Mirror mode: if the file already exists locally and has the same size and modification time as the remote file, it will not be fetched. +Note that the +.Fl m +and +.Fl r +flags are mutually exclusive. .It Fl n Don't preserve the modification time of the transferred file. .It Fl o Ar file @@ -148,6 +153,11 @@ The output files are precious, and should not be deleted under any circumstances, even if the transfer failed or was incomplete. .It Fl r Restart a previously interrupted transfer. +Note that the +.Fl m +and +.Fl r +flags are mutually exclusive. .It Fl S Ar bytes Require the file size reported by the server to match the specified value. diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c index 02a0cd511ed3..99a9cda3a612 100644 --- a/usr.bin/fetch/fetch.c +++ b/usr.bin/fetch/fetch.c @@ -196,6 +196,7 @@ fetch(char *URL, char *path) timeout = 0; *flags = 0; + count = 0; /* common flags */ if (v_level > 2) @@ -232,12 +233,10 @@ fetch(char *URL, char *path) /* set the protocol timeout. */ fetchTimeout = timeout; - /* stat remote file */ - if (fetchStat(url, &us, flags) == -1) - goto failure; - /* just print size */ if (s_flag) { + if (fetchStat(url, &us, flags) == -1) + goto failure; if (us.size == -1) printf("Unknown\n"); else @@ -245,12 +244,40 @@ fetch(char *URL, char *path) goto success; } - /* check that size is as expected */ - if (S_size && us.size != -1 && us.size != S_size) { - warnx("%s: size mismatch: expected %lld, actual %lld", - path, S_size, us.size); + /* + * If the -r flag was specified, we have to compare the local and + * remote files, so we should really do a fetchStat() first, but I + * know of at least one HTTP server that only sends the content + * size in response to GET requests, and leaves it out of replies + * to HEAD requests. Also, in the (frequent) case that the local + * and remote files match but the local file is truncated, we have + * sufficient information *before* the compare to issue a correct + * request. Therefore, we always issue a GET request as if we were + * sure the local file was a truncated copy of the remote file; we + * can drop the connection later if we change our minds. + */ + if (r_flag && !o_stdout && stat(path, &sb) != -1) + url->offset = sb.st_size; + + /* start the transfer */ + if ((f = fetchXGet(url, &us, flags)) == NULL) { + warnx("%s: %s", path, fetchLastErrString); goto failure; } + if (sigint) + goto signal; + + /* check that size is as expected */ + if (S_size) { + if (us.size == -1) { + warnx("%s: size unknown", path); + goto failure; + } else if (us.size != S_size) { + warnx("%s: size mismatch: expected %lld, actual %lld", + path, S_size, us.size); + goto failure; + } + } /* symlink instead of copy */ if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { @@ -261,33 +288,56 @@ fetch(char *URL, char *path) goto success; } + /* open output file */ if (o_stdout) { /* output to stdout */ of = stdout; - } else if (r_flag && us.size != -1 && stat(path, &sb) != -1 - && (F_flag || (us.mtime && sb.st_mtime == us.mtime))) { - /* output to file, restart aborted transfer */ + } else if (url->offset) { + /* resume mode, local file exists */ + if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { + /* no match! have to refetch */ + fclose(f); + url->offset = 0; + if ((f = fetchXGet(url, &us, flags)) == NULL) { + warnx("%s: %s", path, fetchLastErrString); + goto failure; + } + if (sigint) + goto signal; + } else { + us.size += url->offset; + } if (us.size == sb.st_size) + /* nothing to do */ goto success; - else if (sb.st_size > us.size && truncate(path, us.size) == -1) { - warn("%s: truncate()", path); + if (sb.st_size > us.size) { + /* local file too long! */ + warnx("%s: local file (%lld bytes) is longer " + "than remote file (%lld bytes)", + path, sb.st_size, us.size); goto failure; } + /* we got through, open local file in append mode */ + /* + * XXX there's a race condition here - the file we open is not + * necessarily the same as the one we stat()'ed earlier... + */ if ((of = fopen(path, "a")) == NULL) { warn("%s: open()", path); goto failure; } - url->offset = sb.st_size; - } else if (m_flag && us.size != -1 && stat(path, &sb) != -1) { - /* output to file, mirror mode */ + } + if (m_flag && stat(path, &sb) != -1) { + /* mirror mode, local file exists */ if (sb.st_size == us.size && sb.st_mtime == us.mtime) - return 0; - if ((of = fopen(path, "w")) == NULL) { - warn("%s: open()", path); - goto failure; - } - } else { - /* output to file, all other cases */ + goto success; + } + if (!of) { + /* + * We don't yet have an output file; either this is a vanilla + * run with no special flags, or the local and remote files + * didn't match. + */ if ((of = fopen(path, "w")) == NULL) { warn("%s: open()", path); goto failure; @@ -295,14 +345,6 @@ fetch(char *URL, char *path) } count = url->offset; - /* start the transfer */ - if ((f = fetchGet(url, flags)) == NULL) { - warnx("%s", fetchLastErrString); - if (!R_flag && !r_flag && !o_stdout) - unlink(path); - goto failure; - } - /* start the counter */ stat_start(&xs, path, us.size, count); @@ -341,6 +383,7 @@ fetch(char *URL, char *path) } /* timed out or interrupted? */ + signal: if (sigalrm) warnx("transfer timed out"); if (sigint) @@ -472,6 +515,8 @@ main(int argc, char *argv[]) break; case 'M': case 'm': + if (r_flag) + errx(1, "the -m and -r flags are mutually exclusive"); m_flag = 1; break; case 'n': @@ -488,6 +533,8 @@ main(int argc, char *argv[]) R_flag = 1; break; case 'r': + if (m_flag) + errx(1, "the -m and -r flags are mutually exclusive"); r_flag = 1; break; case 'S': @@ -534,7 +581,7 @@ main(int argc, char *argv[]) errx(1, strerror(ENOMEM)); argc++; } - + if (!argc) { usage(); exit(EX_USAGE); @@ -564,8 +611,10 @@ main(int argc, char *argv[]) sa.sa_flags = 0; sa.sa_handler = sig_handler; sigemptyset(&sa.sa_mask); - (void)sigaction(SIGALRM, &sa, NULL); - (void)sigaction(SIGINT, &sa, NULL); + sigaction(SIGALRM, &sa, NULL); + sa.sa_flags = SA_RESETHAND; + sigaction(SIGINT, &sa, NULL); + fetchRestartCalls = 0; /* output file */ if (o_flag) { @@ -614,7 +663,7 @@ main(int argc, char *argv[]) } if (sigint) - exit(1); + kill(getpid(), SIGINT); if (e == 0 && once_flag) exit(0); |