3 files changed, 107 insertions, 60 deletions

diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index 702c6bc1c9f7..fceda252776c 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -342,7 +342,7 @@ and
 .Fn fetchPutHTTP
 functions implement the HTTP/1.1 protocol.
 With a little luck, there's
-even a chance that they comply with RFC2068.
+even a chance that they comply with RFC2616 and RFC2617.
 .Pp
 If the
 .Fa d
@@ -487,6 +487,10 @@ variable.
 This variable is used if and only if connected to an HTTP proxy, and
 is ignored if a user and/or a password were specified in the proxy
 URL.
+.It Ev HTTP_USER_AGENT
+Specifies the User-Agent string to use for HTTP requests.
+This can be useful when working with HTTP origin or proxy servers that
+differentiate between usder agents.
 .El
 .Sh SEE ALSO
 .Xr fetch 1 ,
@@ -520,10 +524,24 @@ URL.
 .%A J. Gettys
 .%A J. Mogul
 .%A H. Frystyk
+.%A L. Masinter
+.%A P. Leach
 .%A T. Berners-Lee
-.%D Januray 1997
+.%D Januray 1999
 .%B Hypertext Transfer Protocol -- HTTP/1.1
-.%O RFC2068
+.%O RFC2616
+.Re
+.Rs
+.%A J. Franks
+.%A P. Hallam-Baker
+.%A J. Hostetler
+.%A S. Lawrence
+.%A P. Leach
+.%A A. Luotonen
+.%A L. Stewart
+.%D June 1999
+.%B HTTP Authentication: Basic and Digest Access Authentication
+.%O RFC2617
 .Re
 .Sh HISTORY
 The
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 0cc9910adaca..f1ffbf2dda0a 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -349,7 +349,7 @@ _ftp_readfn(void *v, char *buf, int len)
 	return r;
     if (r == 0) {
 	io->eof = 1;
-	return _ftp_closefn(v);
+	return 0;
     }
     if (errno != EINTR)
 	io->err = errno;
@@ -712,6 +712,49 @@ ouch:
 }
 
 /*
+ * Authenticate
+ */
+static int
+_ftp_authenticate(int cd, struct url *url, struct url *purl)
+{
+    char *user, *pwd, *logname;
+    char pbuf[MAXHOSTNAMELEN + MAXLOGNAME + 1];
+    int e, len;
+
+    /* XXX FTP_AUTH, and maybe .netrc */
+    
+    /* send user name and password */
+    user = url->user;
+    if (!user || !*user)
+	user = getenv("FTP_LOGIN");
+    if (!user || !*user)
+	user = FTP_ANONYMOUS_USER;
+    if (purl && url->port == _fetch_default_port(url->scheme))
+	e = _ftp_cmd(cd, "USER %s@%s", user, url->host);
+    else if (purl)
+	e = _ftp_cmd(cd, "USER %s@%s@%d", user, url->host, url->port);
+    else
+	e = _ftp_cmd(cd, "USER %s", user);
+    
+    /* did the server request a password? */
+    if (e == FTP_NEED_PASSWORD) {
+	pwd = url->pwd;
+	if (!pwd || !*pwd)
+	    pwd = getenv("FTP_PASSWORD");
+	if (!pwd || !*pwd) {
+	    if ((logname = getlogin()) == 0)
+		logname = FTP_ANONYMOUS_USER;
+	    len = snprintf(pbuf, MAXLOGNAME + 1, "%s@", logname);
+	    gethostname(pbuf + len, sizeof pbuf - len);
+	    pwd = pbuf;
+	}
+	e = _ftp_cmd(cd, "PASS %s", pwd);
+    }
+    
+    return e;
+}
+
+/*
  * Log on to FTP server
  */
 static int
@@ -723,11 +766,6 @@ _ftp_connect(struct url *url, struct url *purl, const char *flags)
 #else
     int af = AF_INET;
 #endif
-    const char *logname;
-    const char *user;
-    const char *pwd;
-    char localhost[MAXHOSTNAMELEN];
-    char pbuf[MAXHOSTNAMELEN + MAXLOGNAME + 1];
 
     direct = CHECK_FLAG('d');
     verbose = CHECK_FLAG('v');
@@ -758,45 +796,11 @@ _ftp_connect(struct url *url, struct url *purl, const char *flags)
     /* expect welcome message */
     if ((e = _ftp_chkerr(cd)) != FTP_SERVICE_READY)
 	goto fouch;
-
-    /* XXX FTP_AUTH, and maybe .netrc */
-    
-    /* send user name and password */
-    user = url->user;
-    if (!user || !*user)
-	user = getenv("FTP_LOGIN");
-    if (!user || !*user)
-	user = FTP_ANONYMOUS_USER;
-    if (purl && url->port == _fetch_default_port(url->scheme))
-	e = _ftp_cmd(cd, "USER %s@%s", user, url->host);
-    else if (purl)
-	e = _ftp_cmd(cd, "USER %s@%s@%d", user, url->host, url->port);
-    else
-	e = _ftp_cmd(cd, "USER %s", user);
     
-    /* did the server request a password? */
-    if (e == FTP_NEED_PASSWORD) {
-	pwd = url->pwd;
-	if (!pwd || !*pwd)
-	    pwd = getenv("FTP_PASSWORD");
-	if (!pwd || !*pwd) {
-	    if ((logname = getlogin()) == 0)
-		logname = FTP_ANONYMOUS_USER;
-	    gethostname(localhost, sizeof localhost);
-	    snprintf(pbuf, sizeof pbuf, "%s@%s", logname, localhost);
-	    pwd = pbuf;
-	}
-	e = _ftp_cmd(cd, "PASS %s", pwd);
-    }
-
-    /* did the server request an account? */
-    if (e == FTP_NEED_ACCOUNT)
+    /* authenticate */
+    if ((e = _ftp_authenticate(cd, url, purl)) != FTP_LOGGED_IN)
 	goto fouch;
     
-    /* we should be done by now */
-    if (e != FTP_LOGGED_IN)
-	goto fouch;
-
     /* might as well select mode and type at once */
 #ifdef FTP_FORCE_STREAM_MODE
     if ((e = _ftp_cmd(cd, "MODE S")) != FTP_OK) /* default is S */
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index b1332b31588c..e1e0721c4d8d 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -294,7 +294,8 @@ typedef enum {
     hdr_content_range,
     hdr_last_modified,
     hdr_location,
-    hdr_transfer_encoding
+    hdr_transfer_encoding,
+    hdr_www_authenticate
 } hdr;
 
 /* Names of interesting headers */
@@ -307,6 +308,7 @@ static struct {
     { hdr_last_modified,	"Last-Modified" },
     { hdr_location,		"Location" },
     { hdr_transfer_encoding,	"Transfer-Encoding" },
+    { hdr_www_authenticate,	"WWW-Authenticate" },
     { hdr_unknown,		NULL },
 };
 
@@ -564,6 +566,8 @@ _http_basic_auth(int fd, const char *hdr, const char *usr, const char *pwd)
     char *upw, *auth;
     int r;
 
+    DEBUG(fprintf(stderr, "usr: [\033[1m%s\033[m]\n", usr));
+    DEBUG(fprintf(stderr, "pwd: [\033[1m%s\033[m]\n", pwd));
     if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
 	return -1;
     auth = _http_base64(upw);
@@ -705,16 +709,16 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
     n = noredirect ? 1 : MAX_REDIRECT;
     i = 0;
 
+    need_auth = 0;
     do {
 	new = NULL;
 	chunked = 0;
-	need_auth = 0;
 	offset = 0;
 	clength = -1;
 	length = -1;
 	size = -1;
 	mtime = 0;
-    retry:
+	
 	/* check port */
 	if (!url->port)
 	    url->port = _fetch_default_port(url->scheme);
@@ -743,6 +747,12 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 		      op, url->doc);
 	}
 
+	/* virtual host */
+	if (url->port == _fetch_default_port(url->scheme))
+	    _http_cmd(fd, "Host: %s", host);
+	else
+	    _http_cmd(fd, "Host: %s:%d", host, url->port);
+	
 	/* proxy authorization */
 	if (purl) {
 	    if (*purl->user || *purl->pwd)
@@ -753,7 +763,7 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 	}
 	
 	/* server authorization */
-	if (need_auth) {
+	if (need_auth || *url->user || *url->pwd) {
 	    if (*url->user || *url->pwd)
 		_http_basic_auth(fd, "Authorization", url->user, url->pwd);
 	    else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0')
@@ -765,11 +775,10 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 	}
 
 	/* other headers */
-	if (url->port == _fetch_default_port(url->scheme))
-	    _http_cmd(fd, "Host: %s", host);
+	if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
+	    _http_cmd(fd, "User-Agent: %s", p);
 	else
-	    _http_cmd(fd, "Host: %s:%d", host, url->port);
-	_http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname);
+	    _http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname);
 	if (url->offset)
 	    _http_cmd(fd, "Range: bytes=%lld-", url->offset);
 	_http_cmd(fd, "Connection: close");
@@ -800,9 +809,7 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 	    /* try again, but send the password this time */
 	    if (verbose)
 		_fetch_info("server requires authorization");
-	    need_auth = 1;
-	    close(fd);
-	    goto retry;
+	    break;
 	case HTTP_NEED_PROXY_AUTH:
 	    /*
 	     * If we're talking to a proxy, we already sent our proxy
@@ -867,6 +874,11 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 		/* XXX weak test*/
 		chunked = (strcasecmp(p, "chunked") == 0);
 		break;
+	    case hdr_www_authenticate:
+		if (code != HTTP_NEED_AUTH)
+		    break;
+		/* if we were smarter, we'd check the method and realm */
+		break;
 	    case hdr_end:
 		/* fall through */
 	    case hdr_unknown:
@@ -875,19 +887,32 @@ _http_request(struct url *URL, const char *op, struct url_stat *us,
 	    }
 	} while (h > hdr_end);
 
-	/* we either have a hit, or a redirect with no Location: header */
-	if (code == HTTP_OK || code == HTTP_PARTIAL || !new)
+	/* we have a hit */
+	if (code == HTTP_OK || code == HTTP_PARTIAL)
 	    break;
 
-	/* we have a redirect */
+	/* we need to provide authentication */
+	if (code == HTTP_NEED_AUTH) {
+	    need_auth = 1;
+	    close(fd);
+	    fd = -1;
+	    continue;
+	}
+	
+	/* all other cases: we got a redirect */
+	need_auth = 0;
 	close(fd);
 	fd = -1;
+	if (!new) {
+	    DEBUG(fprintf(stderr, "redirect with no new location\n"));
+	    break;
+	}
 	if (url != URL)
 	    fetchFreeURL(url);
 	url = new;
     } while (++i < n);
 
-    /* no success */
+    /* we failed, or ran out of retries */
     if (fd == -1) {
 	_http_seterr(code);
 	goto ouch;