4 files changed, 118 insertions, 71 deletions

diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index c5fb45f70da2..c1888a1518a0 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -1403,47 +1403,112 @@ fetch_read(conn_t *conn, void *buf, size_t len)
  */
 #define MIN_BUF_SIZE 1024
 
-int
+ssize_t
 fetch_getln(conn_t *conn)
 {
 	char *tmp;
 	size_t tmpsize;
-	ssize_t len;
-	char c;
+	ssize_t rlen;
 
+	/* allocate initial buffer */
 	if (conn->buf == NULL) {
-		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
-			errno = ENOMEM;
+		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL)
 			return (-1);
-		}
 		conn->bufsize = MIN_BUF_SIZE;
+		conn->buflen = 0;
+		conn->pos = 0;
 	}
 
-	conn->buf[0] = '\0';
-	conn->buflen = 0;
+	/* move residual data up */
+	if (conn->buflen > 0) {
+		if (conn->pos < conn->buflen) {
+			memmove(conn->buf, conn->buf + conn->pos,
+			    conn->buflen - conn->pos);
+		}
+		conn->buflen -= conn->pos;
+		conn->pos -= conn->pos;
+	}
 
-	do {
-		len = fetch_read(conn, &c, 1);
-		if (len == -1)
+	/* do we have a complete line? */
+	while (conn->pos < conn->buflen)
+		if (conn->buf[conn->pos++] == '\n')
+			goto found;
+
+	for (;;) {
+		/* read as much as we can right now */
+		rlen = fetch_read(conn, conn->buf + conn->buflen,
+		    conn->bufsize - conn->buflen - 1);
+		/* error */
+		if (rlen < 0)
 			return (-1);
-		if (len == 0)
+		/* advance and terminate */
+		conn->buflen += rlen;
+		conn->buf[conn->buflen] = '\0';
+		/* connection closed */
+		if (rlen == 0)
 			break;
-		conn->buf[conn->buflen++] = c;
-		if (conn->buflen == conn->bufsize) {
+		/* look for a newline */
+		while (conn->pos < conn->buflen)
+			if (conn->buf[conn->pos++] == '\n')
+				goto found;
+		/* do we need a bigger buffer? */
+		if (conn->buflen > conn->bufsize / 2) {
 			tmp = conn->buf;
-			tmpsize = conn->bufsize * 2 + 1;
-			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
-				errno = ENOMEM;
+			tmpsize = conn->bufsize * 2;
+			if ((tmp = realloc(tmp, tmpsize)) == NULL)
 				return (-1);
-			}
 			conn->buf = tmp;
 			conn->bufsize = tmpsize;
 		}
-	} while (c != '\n');
+	}
+	/* connection closed, return what's left */
+	conn->pos = conn->buflen;
+found:
+	rlen = conn->pos;
+	if (rlen > 0 && conn->buf[rlen - 1] == '\n')
+		conn->buf[--rlen] = '\0';
+	if (rlen > 0 && conn->buf[rlen - 1] == '\r')
+		conn->buf[--rlen] = '\0';
+	DEBUGF("<<< %.*s\n", (int)rlen, conn->buf);
+	return (rlen);
+}
 
-	conn->buf[conn->buflen] = '\0';
-	DEBUGF("<<< %s", conn->buf);
-	return (0);
+
+/*
+ * Read from a connection, taking previously buffered data into account.
+ */
+ssize_t
+fetch_bufread(conn_t *conn, void *buf, size_t len)
+{
+	ssize_t rlen;
+
+	/* avoid overflow */
+	if (len > SSIZE_MAX)
+		len = SSIZE_MAX;
+
+	/* allocate initial buffer */
+	if (conn->buf == NULL) {
+		conn->bufsize = MIN_BUF_SIZE;
+		while (conn->bufsize < len)
+			conn->bufsize *= 2;
+		if ((conn->buf = malloc(conn->bufsize)) == NULL)
+			return (-1);
+		conn->buflen = 0;
+		conn->pos = 0;
+	}
+
+	/* return residual data first */
+	if (conn->buflen > conn->pos) {
+		if (len > conn->buflen - conn->pos)
+			rlen = conn->buflen - conn->pos;
+		else
+			rlen = len;
+		memcpy(buf, conn->buf + conn->pos, rlen);
+		conn->pos += rlen;
+		return (rlen);
+	}
+
+	return (fetch_read(conn, buf, len));
 }
 
 
diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h
index 479c5e03c0d1..5ad9bde513a0 100644
--- a/lib/libfetch/common.h
+++ b/lib/libfetch/common.h
@@ -51,6 +51,7 @@ struct fetchconn {
 	char		*buf;		/* buffer */
 	size_t		 bufsize;	/* buffer size */
 	size_t		 buflen;	/* length of buffer contents */
+	size_t		 pos;		/* current position in buffer */
 	int		 err;		/* last protocol reply code */
 #ifdef WITH_SSL
 	SSL		*ssl;		/* SSL handle */
@@ -120,7 +121,8 @@ int		 fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*);
 #endif
 int		 fetch_ssl(conn_t *, const struct url *, int);
 ssize_t		 fetch_read(conn_t *, void *, size_t);
-int		 fetch_getln(conn_t *);
+ssize_t		 fetch_getln(conn_t *);
+ssize_t		 fetch_bufread(conn_t *, void *, size_t);
 ssize_t		 fetch_write(conn_t *, const void *, size_t);
 ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
 int		 fetch_putln(conn_t *, const char *, size_t);
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 864eacc5d7ff..fc096054c9a2 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -142,24 +142,21 @@ unmappedaddr(struct sockaddr_in6 *sin6)
 static int
 ftp_chkerr(conn_t *conn)
 {
-	if (fetch_getln(conn) == -1) {
+	ssize_t rlen;
+
+	if ((rlen = fetch_getln(conn)) < 0) {
 		fetch_syserr();
 		return (-1);
 	}
 	if (isftpinfo(conn->buf)) {
 		while (conn->buflen && !isftpreply(conn->buf)) {
-			if (fetch_getln(conn) == -1) {
+			if ((rlen = fetch_getln(conn)) < 0) {
 				fetch_syserr();
 				return (-1);
 			}
 		}
 	}
 
-	while (conn->buflen &&
-	    isspace((unsigned char)conn->buf[conn->buflen - 1]))
-		conn->buflen--;
-	conn->buf[conn->buflen] = '\0';
-
 	if (!isftpreply(conn->buf)) {
 		ftp_seterr(FTP_PROTOCOL_ERROR);
 		return (-1);
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index adcc75fcb83a..89f731c896dc 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -148,26 +148,27 @@ struct httpio
 static int
 http_new_chunk(struct httpio *io)
 {
-	char *p;
+	unsigned char *p, *eol;
 
 	if (fetch_getln(io->conn) == -1)
 		return (-1);
 
-	if (io->conn->buflen < 2 || !isxdigit((unsigned char)*io->conn->buf))
+	p = (unsigned char *)io->conn->buf;
+	if (io->conn->pos < 2 || !isxdigit(*p))
 		return (-1);
 
-	for (p = io->conn->buf; *p && !isspace((unsigned char)*p); ++p) {
-		if (*p == ';')
-			break;
-		if (!isxdigit((unsigned char)*p))
+	eol = (unsigned char *)io->conn->buf + io->conn->pos;
+	while (p < eol && *p && !isspace(*p) && *p != ';') {
+		io->chunksize <<= 4;
+		if (*p >= '0' && *p <= '9')
+			io->chunksize += *p - '0';
+		else if (*p >= 'A' && *p <= 'F')
+			io->chunksize += 10 + *p - 'A';
+		else if (*p >= 'a' && *p <= 'f')
+			io->chunksize += 10 + *p - 'a';
+		else
 			return (-1);
-		if (isdigit((unsigned char)*p)) {
-			io->chunksize = io->chunksize * 16 +
-			    *p - '0';
-		} else {
-			io->chunksize = io->chunksize * 16 +
-			    10 + tolower((unsigned char)*p) - 'a';
-		}
+		p++;
 	}
 
 #ifndef NDEBUG
@@ -221,7 +222,7 @@ http_fillbuf(struct httpio *io, size_t len)
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
-		if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
+		if ((nbytes = fetch_bufread(io->conn, io->buf, len)) == -1) {
 			io->error = errno;
 			return (-1);
 		}
@@ -247,7 +248,7 @@ http_fillbuf(struct httpio *io, size_t len)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
 		return (-1);
-	if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
+	if ((nbytes = fetch_bufread(io->conn, io->buf, len)) == -1) {
 		io->error = errno;
 		return (-1);
 	}
@@ -256,8 +257,8 @@ http_fillbuf(struct httpio *io, size_t len)
 	io->chunksize -= nbytes;
 
 	if (io->chunksize == 0) {
-		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
-		    fetch_read(io->conn, &ch, 1) != 1 || ch != '\n')
+		if (fetch_bufread(io->conn, &ch, 1) != 1 || ch != '\r' ||
+		    fetch_bufread(io->conn, &ch, 1) != 1 || ch != '\n')
 			return (-1);
 	}
 
@@ -515,34 +516,19 @@ clean_http_headerbuf(http_headerbuf_t *buf)
 	init_http_headerbuf(buf);
 }
 
-/* Remove whitespace at the end of the buffer */
-static void
-http_conn_trimright(conn_t *conn)
-{
-	while (conn->buflen &&
-	       isspace((unsigned char)conn->buf[conn->buflen - 1]))
-		conn->buflen--;
-	conn->buf[conn->buflen] = '\0';
-}
-
 static hdr_t
 http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
 {
 	unsigned int i, len;
 
-	/*
-	 * Have to do the stripping here because of the first line. So
-	 * it's done twice for the subsequent lines. No big deal
-	 */
-	http_conn_trimright(conn);
-	if (conn->buflen == 0)
+	if (conn->pos == 0 || conn->buf[0] == '\0')
 		return (hdr_end);
 
 	/* Copy the line to the headerbuf */
-	if (hbuf->bufsize < conn->buflen + 1) {
-		if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL)
+	if (hbuf->bufsize < conn->pos + 1) {
+		if ((hbuf->buf = realloc(hbuf->buf, conn->pos + 1)) == NULL)
 			return (hdr_syserror);
-		hbuf->bufsize = conn->buflen + 1;
+		hbuf->bufsize = conn->pos + 1;
 	}
 	strcpy(hbuf->buf, conn->buf);
 	hbuf->buflen = conn->buflen;
@@ -556,12 +542,9 @@ http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
 			return (hdr_syserror);
 
 		/*
-		 * Note: we carry on the idea from the previous version
-		 * that a pure whitespace line is equivalent to an empty
-		 * one (so it's not continuation and will be handled when
-		 * we are called next)
+		 * Note: we previously considered a pure whitespace line
+		 * equivalent to an empty one.  This was incorrect.
 		 */
-		http_conn_trimright(conn);
 		if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
 			break;