4 files changed, 126 insertions, 57 deletions

diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index d29809e69248..723cf3b380ee 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -418,6 +418,14 @@ and message, e.g. "File is not available (404 Not Found)"
 .It Ev FTP_PROXY
 host name of the FTP proxy to use, optionally followed by a port
 number separated from the host name by a colon.
+.Nm libfetch
+will send
+.Ql user@host
+as user name to the proxy, where
+.Ql user
+is the real user name, and
+.Ql host
+is the name of the FTP server.
 .It Ev HTTP_AUTH
 Specifies HTTP authorization parameters, used only if the server
 requires authorization and no user name or password was specified in
@@ -431,10 +439,15 @@ password, in that order.
 host name of the HTTP proxy to use, optionally followed by a port
 number separated from the host name by a colon.
 If no port number is specified, the default is 3128.
+Note that the HTTP proxy will also be used for FTP documents, unless
+the
+.Ev FTP_PROXY
+variable is set.
 .It Ev HTTP_PROXY_AUTH
 Specifies authorization parameters for the HTTP proxy in the same
-format as
-.Ev HTTP_AUTH .
+format as the
+.Ev HTTP_AUTH
+variable.
 The value of this variable is used if and only if connected to an HTTP
 proxy.
 .El
diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c
index cb09efcfeac4..3eb8075fc780 100644
--- a/lib/libfetch/fetch.c
+++ b/lib/libfetch/fetch.c
@@ -80,9 +80,6 @@ fetchXGet(struct url *URL, struct url_stat *us, char *flags)
     else if (strcasecmp(URL->scheme, "http") == 0)
 	return fetchXGetHTTP(URL, us, flags);
     else if (strcasecmp(URL->scheme, "ftp") == 0) {
-	if (!direct &&
-	    getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL)
-	    return fetchXGetHTTP(URL, us, flags);
 	return fetchXGetFTP(URL, us, flags);
     } else {
 	_url_seterr(URL_BAD_SCHEME);
@@ -115,9 +112,6 @@ fetchPut(struct url *URL, char *flags)
     else if (strcasecmp(URL->scheme, "http") == 0)
 	return fetchPutHTTP(URL, flags);
     else if (strcasecmp(URL->scheme, "ftp") == 0) {
-	if (!direct &&
-	    getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL)
-	    return fetchPutHTTP(URL, flags);
 	return fetchPutFTP(URL, flags);
     } else {
 	_url_seterr(URL_BAD_SCHEME);
@@ -140,9 +134,6 @@ fetchStat(struct url *URL, struct url_stat *us, char *flags)
     else if (strcasecmp(URL->scheme, "http") == 0)
 	return fetchStatHTTP(URL, us, flags);
     else if (strcasecmp(URL->scheme, "ftp") == 0) {
-	if (!direct &&
-	    getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL)
-	    return fetchStatHTTP(URL, us, flags);
 	return fetchStatFTP(URL, us, flags);
     } else {
 	_url_seterr(URL_BAD_SCHEME);
@@ -165,9 +156,6 @@ fetchList(struct url *URL, char *flags)
     else if (strcasecmp(URL->scheme, "http") == 0)
 	return fetchListHTTP(URL, flags);
     else if (strcasecmp(URL->scheme, "ftp") == 0) {
-	if (!direct &&
-	    getenv("FTP_PROXY") == NULL && getenv("HTTP_PROXY") != NULL)
-	    return fetchListHTTP(URL, flags);
 	return fetchListFTP(URL, flags);
     } else {
 	_url_seterr(URL_BAD_SCHEME);
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 05f429bd9119..a5eb1d351734 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -594,7 +594,7 @@ _ftp_connect(char *host, int port, char *user, char *pwd, char *flags)
 	af = AF_INET6;
 
     /* check for proxy */
-    if (!direct && (p = getenv("FTP_PROXY")) != NULL) {
+    if (!direct && (p = getenv("FTP_PROXY")) != NULL && *p) {
 	char c = 0;
 
 #ifdef INET6
@@ -651,8 +651,12 @@ _ftp_connect(char *host, int port, char *user, char *pwd, char *flags)
     /* send user name and password */
     if (!user || !*user)
 	user = FTP_ANONYMOUS_USER;
-    e = p ? _ftp_cmd(cd, "USER %s@%s@%d", user, host, port)
-	  : _ftp_cmd(cd, "USER %s", user);
+    if (p && port == FTP_DEFAULT_PORT)
+	e = _ftp_cmd(cd, "USER %s@%s", user, host);
+    else if (p)
+	e = _ftp_cmd(cd, "USER %s@%s@%d", user, host, port);
+    else
+	e = _ftp_cmd(cd, "USER %s", user);
     
     /* did the server request a password? */
     if (e == FTP_NEED_PASSWORD) {
@@ -759,12 +763,26 @@ _ftp_cached_connect(struct url *url, char *flags)
 }
 
 /*
+ * Check to see if we should use an HTTP proxy instead
+ */
+static int
+_ftp_use_http_proxy(void)
+{
+    char *p;
+
+    return ((p = getenv("HTTP_PROXY")) && *p && !getenv("FTP_PROXY"));
+}
+
+/*
  * Get and stat file
  */
 FILE *
 fetchXGetFTP(struct url *url, struct url_stat *us, char *flags)
 {
     int cd;
+
+    if (_ftp_use_http_proxy())
+	return fetchXGetHTTP(url, us, flags);
     
     /* connect to server */
     if ((cd = _ftp_cached_connect(url, flags)) == NULL)
@@ -800,6 +818,9 @@ fetchPutFTP(struct url *url, char *flags)
 {
     int cd;
 
+    if (_ftp_use_http_proxy())
+	return fetchPutHTTP(url, flags);
+    
     /* connect to server */
     if ((cd = _ftp_cached_connect(url, flags)) == NULL)
 	return NULL;
@@ -821,6 +842,9 @@ fetchStatFTP(struct url *url, struct url_stat *us, char *flags)
 {
     int cd;
 
+    if (_ftp_use_http_proxy())
+	return fetchStatHTTP(url, us, flags);
+    
     us->size = -1;
     us->atime = us->mtime = 0;
     
@@ -843,6 +867,9 @@ extern void warnx(char *, ...);
 struct url_ent *
 fetchListFTP(struct url *url, char *flags)
 {
+    if (_ftp_use_http_proxy())
+	return fetchListHTTP(url, flags);
+    
     warnx("fetchListFTP(): not implemented");
     return NULL;
 }
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 06dad22ade4a..090114283286 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -419,54 +419,70 @@ _http_next_header(int fd, char **p)
 /*
  * Parse a last-modified header
  */
-static time_t
-_http_parse_mtime(char *p)
+static int
+_http_parse_mtime(char *p, time_t *mtime)
 {
-    char locale[64];
+    char locale[64], *r;
     struct tm tm;
 
     strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
     setlocale(LC_TIME, "C");
-    strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+    r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
     /* XXX should add support for date-2 and date-3 */
     setlocale(LC_TIME, locale);
+    if (r == NULL)
+	return -1;
     DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
 		  "%02d:%02d:%02d\033[m]\n",
 		  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
 		  tm.tm_hour, tm.tm_min, tm.tm_sec));
-    return timegm(&tm);
+    *mtime = timegm(&tm);
+    return 0;
 }
 
 /*
  * Parse a content-length header
  */
-static off_t
-_http_parse_length(char *p)
+static int
+_http_parse_length(char *p, size_t *length)
 {
-    off_t len;
+    size_t len;
     
     for (len = 0; *p && isdigit(*p); ++p)
 	len = len * 10 + (*p - '0');
-    DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", len));
-    return len;
+    DEBUG(fprintf(stderr, "content length: [\033[1m%d\033[m]\n", len));
+    *length = len;
+    return 0;
 }
 
 /*
  * Parse a content-range header
  */
-static off_t
-_http_parse_range(char *p)
+static int
+_http_parse_range(char *p, off_t *offset, size_t *length, size_t *size)
 {
-    off_t off;
-    
+    int first, last, len;
+
     if (strncasecmp(p, "bytes ", 6) != 0)
 	return -1;
-    for (p += 6, off = 0; *p && isdigit(*p); ++p)
-	off = off * 10 + *p - '0';
+    for (first = 0, p += 6; *p && isdigit(*p); ++p)
+	first = first * 10 + *p - '0';
     if (*p != '-')
 	return -1;
-    DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", off));
-    return off;
+    for (last = 0, ++p; *p && isdigit(*p); ++p)
+	last = last * 10 + *p - '0';
+    if (first > last || *p != '/')
+	return -1;
+    for (len = 0, ++p; *p && isdigit(*p); ++p)
+	len = len * 10 + *p - '0';
+    if (len < last - first + 1)
+	return -1;
+    DEBUG(fprintf(stderr, "content range: [\033[1m%d-%d/%d\033[m]\n",
+		  first, last, len));
+    *offset = first;
+    *length = last - first + 1;
+    *size = len;
+    return 0;
 }
 
 
@@ -715,7 +731,7 @@ _http_connect(struct url *URL, int *proxy, char *flags)
 		URL->port = 80;
     }
     
-    if (!direct && (p = getenv("HTTP_PROXY")) != NULL) {
+    if (!direct && (p = getenv("HTTP_PROXY")) != NULL && *p != '\0') {
 	/* attempt to connect to proxy server */
 	if ((fd = _http_proxy_connect(p, af, verbose)) == -1)
 	    return -1;
@@ -751,6 +767,8 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
     int chunked, need_auth, noredirect, proxy, verbose;
     int code, fd, i, n;
     off_t offset;
+    size_t clength, length, size;
+    time_t mtime;
     char *p;
     FILE *f;
     hdr h;
@@ -762,23 +780,22 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
     noredirect = (flags && strchr(flags, 'A'));
     verbose = (flags && strchr(flags, 'v'));
 
+    /* try the provided URL first */
+    url = URL;
+
+    /* if the A flag is set, we only get one try */
     n = noredirect ? 1 : MAX_REDIRECT;
+    i = 0;
 
-    /* just to appease compiler warnings */
-    code = HTTP_PROTOCOL_ERROR;
-    chunked = 0;
-    offset = 0;
-    fd = -1;
-    
-    for (url = URL, i = 0; i < n; ++i) {
+    do {
 	new = NULL;
-	if (us) {
-	    us->size = -1;
-	    us->atime = us->mtime = 0;
-	}
 	chunked = 0;
 	need_auth = 0;
 	offset = 0;
+	clength = -1;
+	length = -1;
+	size = -1;
+	mtime = 0;
     retry:
 	/* connect to server or proxy */
 	if ((fd = _http_connect(url, &proxy, flags)) == -1)
@@ -805,7 +822,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	}
 
 	/* proxy authorization */
-	if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL)
+	if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0')
 	    _http_authorize(fd, "Proxy-Authorization", p);
 	
 	/* server authorization */
@@ -814,7 +831,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 		_http_basic_auth(fd, "Authorization",
 				 url->user ? url->user : "",
 				 url->pwd ? url->pwd : "");
-	    else if ((p = getenv("HTTP_AUTH")) != NULL)
+	    else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0')
 		_http_authorize(fd, "Authorization", p);
 	    else {
 		_http_seterr(HTTP_NEED_AUTH);
@@ -885,15 +902,13 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 		_http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	    case hdr_content_length:
-		if (us)
-		    us->size = _http_parse_length(p);
+		_http_parse_length(p, &clength);
 		break;
 	    case hdr_content_range:
-		offset = _http_parse_range(p);
+		_http_parse_range(p, &offset, &length, &size);
 		break;
 	    case hdr_last_modified:
-		if (us)
-		    us->atime = us->mtime = _http_parse_mtime(p);
+		_http_parse_mtime(p, &mtime);
 		break;
 	    case hdr_location:
 		if (!HTTP_REDIRECT(code))
@@ -942,7 +957,7 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	if (url != URL)
 	    fetchFreeURL(url);
 	url = new;
-    }
+    } while (++i < n);
 
     /* no success */
     if (fd == -1) {
@@ -950,14 +965,40 @@ _http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
 	goto ouch;
     }
 
+    DEBUG(fprintf(stderr, "offset: %lld, length: %d, size: %d, clength: %d\n",
+		  offset, length, size, clength));
+    
+    /* check for inconsistencies */
+    if (clength != -1 && length != -1 && clength != length) {
+	_http_seterr(HTTP_PROTOCOL_ERROR);
+	goto ouch;
+    }
+    if (clength == -1)
+	clength = length;
+    if (clength != -1)
+	length = offset + clength;
+    if (length != -1 && size != -1 && length != size) {
+	_http_seterr(HTTP_PROTOCOL_ERROR);
+	goto ouch;
+    }
+    if (size == -1)
+	size = length;
+
+    /* fill in stats */
+    if (us) {
+	us->size = size;
+	us->atime = us->mtime = mtime;
+    }
+    
     /* too far? */
-    if (offset > url->offset) {
+    if (offset > URL->offset) {
 	_http_seterr(HTTP_PROTOCOL_ERROR);
 	goto ouch;
     }
 
-    /* report back real offset */
+    /* report back real offset and size */
     URL->offset = offset;
+    URL->length = clength;
     
     /* wrap it up in a FILE */
     if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) {