5 files changed, 199 insertions, 66 deletions

diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1
index 1e88778c3f4a..58b0ff7a35d4 100644
--- a/usr.bin/fetch/fetch.1
+++ b/usr.bin/fetch/fetch.1
@@ -1,4 +1,4 @@
-.\" $Id: fetch.1,v 1.9.2.3 1997/03/06 07:21:40 mpp Exp $
+.\" $Id: fetch.1,v 1.9.2.4 1997/03/10 07:12:48 fenner Exp $
 .Dd July 2, 1996
 .Dt FETCH 1
 .Os FreeBSD 2.2
@@ -7,7 +7,7 @@
 .Nd retrieve a file by Uniform Resource Locator
 .Sh SYNOPSIS
 .Nm fetch
-.Op Fl MPamnpqr
+.Op Fl MPabmnpqr
 .Op Fl o Ar file
 .Ar URL
 .Op Ar ...
@@ -48,6 +48,12 @@ The following options are available:
 .Bl -tag -width Fl
 .It Fl a
 Automatically retry the transfer upon soft failures.
+.It Fl b
+Work around a bug in some
+.Tn HTTP
+servers which fail to correctly implement the
+.Tn TCP
+protocol.
 .It Fl c Ar dir
 The file to retrieve is in directory
 .Ar dir
@@ -301,3 +307,7 @@ Only the
 authentication mode is implemented for
 .Tn HTTP .
 This should be replaced by digest authentication.
+.Pp
+The
+.Fl b
+flag should not be necessary.
diff --git a/usr.bin/fetch/fetch.h b/usr.bin/fetch/fetch.h
index 321af11f671b..d885b23999a3 100644
--- a/usr.bin/fetch/fetch.h
+++ b/usr.bin/fetch/fetch.h
@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: fetch.h,v 1.2 1997/01/31 19:55:49 wollman Exp $
+ *	$Id: fetch.h,v 1.3 1997/02/05 19:59:10 wollman Exp $
  */
 
 #ifndef fetch_h
@@ -49,6 +49,7 @@ struct fetch_state {
 	int fs_linkfile;	/* -l option */
 	int fs_precious;	/* -R option */
 	int fs_auto_retry;	/* -a option */
+	int fs_linux_bug;	/* -b option */
 	time_t fs_modtime;
 	void *fs_proto;
 	int (*fs_retrieve)(struct fetch_state *);
diff --git a/usr.bin/fetch/ftp.c b/usr.bin/fetch/ftp.c
index b02b503d1f78..19332d12b3b0 100644
--- a/usr.bin/fetch/ftp.c
+++ b/usr.bin/fetch/ftp.c
@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: ftp.c,v 1.3.2.1 1997/03/10 07:12:49 fenner Exp $
+ *	$Id: ftp.c,v 1.3.2.2 1997/03/11 20:09:49 joerg Exp $
  */
 
 #include <sys/types.h>
@@ -66,8 +66,8 @@ struct uri_scheme ftp_scheme =
 static int 
 ftp_parse(struct fetch_state *fs, const char *uri)
 {
-	const char *p, *colon, *slash, *q;
-	char *hostname, *atsign;
+	const char *p, *slash, *q;
+	char *hostname, *atsign, *colon;
 	unsigned port;
 	struct ftp_state *ftps;
 
@@ -80,29 +80,30 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 	}
 
 	p += 2;
-	colon = strchr(p, ':');
 	slash = strchr(p, '/');
-	if (colon && slash && colon < slash)
-		q = colon;
-	else
-		q = slash;
-	if (q == 0) {
+	if (slash == 0) {
 		warnx("`%s': malformed `ftp' URL", uri);
 		return EX_USAGE;
 	}
-	hostname = alloca(q - p + 1);
+	hostname = alloca(slash - p + 1);
 	hostname[0] = '\0';
-	strncat(hostname, p, q - p);
-	p = slash;
+	strncat(hostname, p, slash - p);
+
+	if ((atsign = strrchr(hostname, '@')) == 0)
+		q = hostname;
+	else
+		q = atsign + 1;
 
-	if (colon && colon < slash && colon + 1 != slash) {
+	if ((colon = strchr(q, ':')) != 0)
+		*colon = '\0';
+
+	if (colon && *(colon + 1)) {
 		unsigned long ul;
 		char *ep;
 
 		errno = 0;
 		ul = strtoul(colon + 1, &ep, 10);
-		if (ep != slash || ep == colon + 1 || errno != 0
-		    || ul < 1 || ul > 65534) {
+		if (*ep || errno != 0 || ul < 1 || ul > 65534) {
 			if (errno)
 				warn("`%s': invalid port in URL", uri);
 			else
@@ -118,6 +119,8 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 	p = slash + 1;
 
 	ftps = safe_malloc(sizeof *ftps);
+	ftps->ftp_password = 0;
+	ftps->ftp_user = 0;
 
 	/*
 	 * Now, we have a copy of the hostname in hostname, the specified port
@@ -125,7 +128,6 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 	 * of the URI.  We just need to check for a user in the hostname,
 	 * and then save all the bits in our state.
 	 */
-	atsign = strrchr(hostname, '@');
 	if (atsign) {
 		if (atsign[1] == '\0') {
 			warnx("`%s': malformed `ftp' hostname", hostname);
@@ -134,12 +136,19 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 		}
 
 		*atsign = '\0';
+		if ((colon = strchr(hostname, ':')) != 0)
+			*colon = '\0';
+		if (hostname[0] == '\0') {
+			warnx("`%s': malformed `ftp' user", atsign + 1);
+			free(ftps);
+			return EX_USAGE;
+		}
+		if (colon != 0)
+			ftps->ftp_password = percent_decode(colon + 1);
 		ftps->ftp_user = percent_decode(hostname);
 		ftps->ftp_hostname = safe_strdup(atsign + 1);
-	} else {
-		ftps->ftp_user = 0;
+	} else
 		ftps->ftp_hostname = safe_strdup(hostname);
-	}
 	ftps->ftp_port = port;
 
 	p = ftps->ftp_remote_file = percent_decode(p);
@@ -150,7 +159,8 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 		fs->fs_outputfile = slash ? slash + 1 : p;
 	}
 
-	ftps->ftp_password = getenv("FTP_PASSWORD");
+	if (ftps->ftp_password == 0)
+		ftps->ftp_password = getenv("FTP_PASSWORD");
 	if (ftps->ftp_password != 0) {
 		ftps->ftp_password = safe_strdup(ftps->ftp_password);
 	} else {
@@ -170,11 +180,10 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 		setenv("FTP_PASSWORD", pw, 0); /* cache the result */
 	}
 
-	if (ftps->ftp_user == 0) {
-		const char *user = getenv("FTP_LOGIN");
-		if (user != 0)
-			ftps->ftp_user = safe_strdup(user);
-	}
+	if (ftps->ftp_user == 0)
+		ftps->ftp_user = getenv("FTP_LOGIN");
+	if (ftps->ftp_user != 0)
+		ftps->ftp_user = safe_strdup(ftps->ftp_user);
 
 	fs->fs_proto = ftps;
 	fs->fs_close = ftp_close;
diff --git a/usr.bin/fetch/http.c b/usr.bin/fetch/http.c
index 2b50fc5d6365..b3f5b4dfd85a 100644
--- a/usr.bin/fetch/http.c
+++ b/usr.bin/fetch/http.c
@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: http.c,v 1.4 1997/02/11 20:46:05 wollman Exp $
+ *	$Id: http.c,v 1.4.2.1 1997/03/10 07:12:51 fenner Exp $
  */
 
 #include <sys/types.h>
@@ -113,6 +113,10 @@ static char *format_http_user_agent(void);
 static enum http_header http_parse_header(char *line, char **valuep);
 static int check_md5(FILE *fp, char *base64ofmd5);
 static int http_first_line(const char *line);
+static int http_suck(struct fetch_state *fs, FILE *remote, FILE *local,
+		     off_t total_length, int timo);
+static int http_suck_chunked(struct fetch_state *fs, FILE *remote, FILE *local,
+			     off_t total_length, int timo);
 static int parse_http_content_range(char *orig, off_t *first, off_t *total);
 static int process_http_auth(struct fetch_state *fs, char *hdr, int autherr);
 static struct http_auth *find_http_auth(struct http_auth_head *list,
@@ -243,7 +247,7 @@ out:
 		return rv;
 	}
 
-	if (strncmp(uri, "http://", 7) == 0) {
+	if (strncmp(uri, "http://", 7) == 0 || strncmp(uri, "ftp://", 6) == 0) {
 		char *hosthdr;
 		slash = strchr(uri + 7, '/');
 		if (slash == 0) {
@@ -418,12 +422,11 @@ http_retrieve(struct fetch_state *fs)
 	int timo;
 	char *line, *new_location;
 	char *errstr = 0;
-	size_t linelen, readresult, writeresult;
+	size_t linelen, writeresult;
 	off_t total_length, restart_from;
 	time_t last_modified, when_to_retry;
 	char *base64ofmd5;
-	static char buf[BUFFER_SIZE];
-	int to_stdout, restarting, redirection, retrying, autherror;
+	int to_stdout, restarting, redirection, retrying, autherror, chunked;
 	char rangebuf[sizeof("Range: bytes=18446744073709551616-\r\n")];
 
 	setup_http_auth();
@@ -433,7 +436,6 @@ http_retrieve(struct fetch_state *fs)
 	restarting = fs->fs_restart;
 	redirection = 0;
 	retrying = 0;
-	autherror = 0;
 
 	/*
 	 * Figure out the timeout.  Prefer the -T command-line value,
@@ -483,7 +485,7 @@ http_retrieve(struct fetch_state *fs)
 	n = 0;
 	msg.msg_control = 0;
 	msg.msg_controllen = 0;
-	msg.msg_flags = MSG_EOF;
+	msg.msg_flags = fs->fs_linux_bug ? 0 : MSG_EOF;
 
 #define addstr(Iov, N, Str) \
 	do { \
@@ -525,9 +527,13 @@ retry:
 			addstr(iov, n, "If-Modified-Since: ");
 			addstr(iov, n, format_http_date(stab.st_mtime));
 			addstr(iov, n, "\r\n");
-		} else if (errno != 0) {
-			warn("%s: cannot mirror; will retrieve anew", 
-			     fs->fs_outputfile);
+		} else if (errno != 0 || !S_ISREG(stab.st_mode)) {
+			if (errno != 0)
+				warn("%s", fs->fs_outputfile);
+			else
+				warnx("%s: not a regular file",
+				      fs->fs_outputfile);
+			warnx("cannot mirror; will retrieve anew");
 		}
 	}
 	if (restarting) {
@@ -543,14 +549,14 @@ retry:
 			sprintf(rangebuf, "Range: bytes=%qd-\r\n", 
 				(quad_t)stab.st_size);
 			addstr(iov, n, rangebuf);
-		} else if (errno != 0) {
-			warn("%s: cannot restart; will retrieve anew",
-			     fs->fs_outputfile);
-			restarting = 0;
-		} else {
-			warnx("%s: cannot restart; will retrieve anew",
-			      fs->fs_outputfile);
+		} else if (errno != 0 || !S_ISREG(stab.st_mode)) {
+			if (errno != 0)
+				warn("%s", fs->fs_outputfile);
+			else
+				warnx("%s: not a regular file",
+				      fs->fs_outputfile);
 			restarting = 0;
+			warnx("cannot restart; will retrieve anew");
 		}
 	}
 	addstr(iov, n, "\r\n");
@@ -575,7 +581,7 @@ retry:
 	fs->fs_status = "sending request message";
 	setup_sigalrm();
 	alarm(timo);
-	if (sendmsg(s, &msg, MSG_EOF) < 0) {
+	if (sendmsg(s, &msg, fs->fs_linux_bug ? 0 : MSG_EOF) < 0) {
 		warn("sendmsg: %s", https->http_hostname);
 		fclose(remote);
 		return EX_OSERR;
@@ -653,6 +659,7 @@ got100reply:
 	 * OK.  The other end is doing HTTP 1.0 at the very least.
 	 * This means that some of the fancy stuff is at least possible.
 	 */
+	autherror = 0;
 	line[linelen - 1] = '\0'; /* turn line into a string */
 	status = http_first_line(line);
 
@@ -717,6 +724,7 @@ got100reply:
 	base64ofmd5 = 0;
 	new_location = 0;
 	restart_from = 0;
+	chunked = 0;
 	fs->fs_status = "parsing reply headers";
 
 	while((line = fgetln(remote, &linelen)) != 0) {
@@ -784,7 +792,11 @@ doretry:
 			break;
 
 		case ht_transfer_encoding:
-			warnx("%s: %s specified a Transfer-Encoding: %s",
+			if (strncasecmp(value, "chunked", 7) == 0) {
+				chunked = 1;
+				break;
+			}
+			warnx("%s: %s specified Transfer-Encoding `%s'",
 			      fs->fs_outputfile, https->http_hostname,
 			      value);
 			warnx("%s: output file may be uninterpretable",
@@ -915,22 +927,13 @@ spewerror:
 	fseek(local, restart_from, SEEK_SET); /* XXX truncation off_t->long */
 	display(fs, total_length, restart_from); /* XXX truncation */
 
-	/*
-	 * Eventually this loop will be separated out as http_suck(), and
-	 * there will be a separate http_suck_chunked() to deal with that
-	 * Transfer-Encoding.
-	 */
-	do {
-		alarm(timo);
-		readresult = fread(buf, 1, sizeof buf, remote);
-		alarm(0);
-
-		if (readresult == 0)
-			break;
-		display(fs, total_length, readresult);
-
-		writeresult = fwrite(buf, 1, readresult, local);
-	} while (writeresult == readresult);
+	if (chunked)
+		status = http_suck_chunked(fs, remote, local, total_length, 
+					   timo);
+	else
+		status = http_suck(fs, remote, local, total_length, timo);
+	if (status)
+		goto out;
 
 	status = errno;		/* save errno for warn(), below, if needed */
 	display(fs, total_length, -1); /* do here in case we have to warn */
@@ -979,6 +982,111 @@ cantauth:
 }
 
 /*
+ * Suck over an HTTP body in standard form.
+ */
+static int
+http_suck(struct fetch_state *fs, FILE *remote, FILE *local, 
+	  off_t total_length, int timo)
+{
+	static char buf[BUFFER_SIZE];
+	ssize_t readresult, writeresult;
+
+	do {
+		alarm(timo);
+		readresult = fread(buf, 1, sizeof buf, remote);
+		alarm(0);
+
+		if (readresult == 0)
+			return 0;
+		display(fs, total_length, readresult);
+
+		writeresult = fwrite(buf, 1, readresult, local);
+	} while (writeresult == readresult);
+	return 0;
+}
+
+/*
+ * Suck over an HTTP body in chunked form.  Ick.
+ * Note that the return value convention here is a bit strange.
+ * A zero return does not necessarily mean success; rather, it means
+ * that this routine has already taken care of error reporting and
+ * just wants to exit.
+ */
+static int
+http_suck_chunked(struct fetch_state *fs, FILE *remote, FILE *local,
+		  off_t total_length, int timo)
+{
+	static char buf[BUFFER_SIZE];
+	ssize_t readresult, writeresult;
+	size_t linelen;
+	u_long chunklen;
+	char *line, *ep;
+
+	for (;;) {
+		alarm(timo);
+		line = fgetln(remote, &linelen);
+		alarm(0);
+		if (line == 0) {
+			warnx("%s: error processing chunked encoding: "
+			      "missing length", fs->fs_outputfile);
+			return EX_PROTOCOL;
+		}
+		line[--linelen] = '\0';
+		for (; linelen > 0; linelen--) {
+			if (isspace(line[linelen - 1]))
+				line[linelen - 1] = '\0';
+		}
+		errno = 0;
+		chunklen = strtoul(line, &ep, 16);
+		if (errno || *line == 0
+		    || (*ep && !isspace(*ep) && *ep != ';')) {
+			warnx("%s: error processing chunked encoding: "
+			      "uninterpretable length: %s", line);
+			return EX_PROTOCOL;
+		}
+		if (chunklen == 0)
+			break;
+
+#ifndef MIN
+#define MIN(a,b) ((a)>(b)?(b):(a))
+#endif
+		while (chunklen > 0) {
+			alarm(timo);
+			readresult = fread(buf, 1, MIN(sizeof buf, chunklen),
+					   remote);
+			alarm(0);
+			if (readresult == 0) {
+				warnx("%s: EOF with %lu left in chunk",
+				      fs->fs_outputfile, chunklen);
+				return EX_PROTOCOL;
+			}
+			display(fs, total_length, readresult);
+			chunklen -= readresult;
+
+			writeresult = fwrite(buf, 1, readresult, local);
+			if (writeresult != readresult)
+				return 0; /* main code will diagnose */
+		}
+		/*
+		 * Read the bogus CRLF after the chunk's body.
+		 */
+		alarm(timo);
+		fread(buf, 1, 2, remote);
+		alarm(0);
+	}
+	/*
+	 * If we got here, then we successfully read every chunk and got
+	 * the end-of-chunks indicator.  Now we have to ignore any trailer
+	 * lines which come across---or we would if we cared about keeping
+	 * the connection open.  Since we are just going to close it anyway,
+	 * we won't bother with that here.  If ever something important is
+	 * defined for the trailer, we will have to revisit that decision.
+	 */
+	return 0;
+}
+
+
+/*
  * The format of the response line for an HTTP request is:
  *	HTTP/V.vv{WS}999{WS}Explanatory text for humans to read\r\n
  * Old pre-HTTP/1.0 servers can return
diff --git a/usr.bin/fetch/main.c b/usr.bin/fetch/main.c
index ff6fb8df7fb9..2913a0ccb802 100644
--- a/usr.bin/fetch/main.c
+++ b/usr.bin/fetch/main.c
@@ -24,7 +24,7 @@
  * SUCH DAMAGE.
  */
 
-/* $Id: main.c,v 1.26.2.4 1997/03/06 07:21:40 mpp Exp $ */
+/* $Id: main.c,v 1.26.2.5 1997/07/02 06:25:28 charnier Exp $ */
 
 #include <sys/types.h>
 
@@ -73,7 +73,7 @@ main(int argc, char *const *argv)
     fs.fs_verbose = 1;
     change_to_dir = file_to_get = hostname = 0;
 
-    while ((c = getopt(argc, argv, "ac:D:f:h:HilLmMnNo:pPqRrT:vV:")) != -1) {
+    while ((c = getopt(argc, argv, "abc:D:f:h:HilLmMnNo:pPqRrT:vV:")) != -1) {
 	    switch (c) {
 	    case 'D': case 'H': case 'I': case 'N': case 'L': case 'V': 
 		    break;	/* ncftp compatibility */
@@ -81,10 +81,15 @@ main(int argc, char *const *argv)
 	    case 'a':
 		    fs.fs_auto_retry = 1;
 		    break;
+
+	    case 'b':
+		    fs.fs_linux_bug = 1;
+		    break;
+
 	    case 'c':
 		    change_to_dir = optarg;
 		    break;
-	    
+
 	    case 'f':
 		    file_to_get = optarg;
 		    break;