From 4cb272a0796b9603c3347ee83fc8ddc1216e7ca0 Mon Sep 17 00:00:00 2001 From: Bryan Drewery Date: Tue, 19 Nov 2013 15:35:26 +0000 Subject: Support SNI in libfetch SNI is Server Name Indentification which is a protocol for TLS that indicates the host that is being connected to at the start of the handshake. It allows to use Virtual Hosts on HTTPS. Submitted by: sbz Submitted by: Michael Gmelin [1] PR: kern/183583 [1] Reviewed by: des Approved by: bapt MFC after: 1 week --- lib/libfetch/common.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 15d5a25de83b..b15ab9ac39e8 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -829,6 +829,15 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) return (-1); } SSL_set_fd(conn->ssl, conn->sd); + +#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) + if (!SSL_set_tlsext_host_name(conn->ssl, URL->host)) { + fprintf(stderr, + "TLS server name indication extension failed for host %s\n", + URL->host); + return (-1); + } +#endif while ((ret = SSL_connect(conn->ssl)) == -1) { ssl_err = SSL_get_error(conn->ssl, ret); if (ssl_err != SSL_ERROR_WANT_READ && -- cgit v1.3 From 5313966ea6124036371626cf0fb41a202916e450 Mon Sep 17 00:00:00 2001 From: Bryan Drewery Date: Tue, 19 Nov 2013 16:11:03 +0000 Subject: Fix build with GCC SSL_set_tlsext_host_name(3) internally does not modify the host buffer pased to it. So it is safe to DECONST the struct url* here. Reported by: gjb Approved by: bapt (implicit) MFC after: 1 week X-MFC-With: r258347 --- lib/libfetch/common.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index b15ab9ac39e8..49d268ce3761 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -831,7 +831,8 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) SSL_set_fd(conn->ssl, conn->sd); #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) - if (!SSL_set_tlsext_host_name(conn->ssl, URL->host)) { + if (!SSL_set_tlsext_host_name(conn->ssl, + __DECONST(struct url *, URL)->host)) { fprintf(stderr, "TLS server name indication extension failed for host %s\n", URL->host); -- cgit v1.3 From 3443b2bc31fe57510c9791698e63724da7df496d Mon Sep 17 00:00:00 2001 From: Dag-Erling Smørgrav Date: Mon, 20 Jan 2014 11:13:05 +0000 Subject: Fix format string. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Submitted by: Jörg Sonnenberger MFC after: 1 week --- lib/libfetch/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 49d268ce3761..fbacb0282a75 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -679,7 +679,7 @@ fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose) if (getenv("SSL_NO_TLS1") != NULL) ssl_ctx_options |= SSL_OP_NO_TLSv1; if (verbose) - fetch_info("SSL options: %x", ssl_ctx_options); + fetch_info("SSL options: %lx", ssl_ctx_options); SSL_CTX_set_options(ctx, ssl_ctx_options); } -- cgit v1.3 From 215a27f1a4aaa9e409f9f362f1725aa40dea53af Mon Sep 17 00:00:00 2001 From: Dag-Erling Smørgrav Date: Tue, 28 Jan 2014 12:48:17 +0000 Subject: Solve http buffering issues and hangs once and for all (hopefully!) by simply not trying to return exactly what the caller asked for - just return whatever we got and let the caller be the judge of whether it was enough. If an error occurs or the connection times out after we already received some data, return a short read, under the assumption that the next call will fail or time out before we read anything. As it turns out, none of the code that calls fetch_read() assumes an all-or-nothing result anyway, except for a couple of lines where we read the CR LF at the end of a hunk in HTTP hunked encoding, so the changes outside of fetch_read() and http_readfn() are minimal. While there, replace select(2) with poll(2). MFC after: 3 days --- lib/libfetch/common.c | 133 +++++++++++++++----------------------------------- lib/libfetch/common.h | 7 --- lib/libfetch/http.c | 38 +++++++-------- 3 files changed, 57 insertions(+), 121 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index fbacb0282a75..ab81b5821f73 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -641,7 +642,7 @@ fetch_ssl_verify_hname(X509 *cert, const char *host) struct addrinfo *ip; STACK_OF(GENERAL_NAME) *altnames; X509_NAME *subject; - int ret; + int ret; ret = 0; ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); @@ -913,33 +914,6 @@ fetch_ssl_read(SSL *ssl, char *buf, size_t len) } #endif -/* - * Cache some data that was read from a socket but cannot be immediately - * returned because of an interrupted system call. - */ -static int -fetch_cache_data(conn_t *conn, char *src, size_t nbytes) -{ - char *tmp; - - if (conn->cache.size < nbytes) { - tmp = realloc(conn->cache.buf, nbytes); - if (tmp == NULL) { - fetch_syserr(); - return (-1); - } - conn->cache.buf = tmp; - conn->cache.size = nbytes; - } - - memcpy(conn->cache.buf, src, nbytes); - conn->cache.len = nbytes; - conn->cache.pos = 0; - - return (0); -} - - static ssize_t fetch_socket_read(int sd, char *buf, size_t len) { @@ -962,46 +936,31 @@ ssize_t fetch_read(conn_t *conn, char *buf, size_t len) { struct timeval now, timeout, delta; - fd_set readfds; - ssize_t rlen, total; - char *start; + struct pollfd pfd; + ssize_t rlen; + int deltams; if (fetchTimeout > 0) { gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } - total = 0; - start = buf; + deltams = INFTIM; + memset(&pfd, 0, sizeof pfd); + pfd.fd = conn->sd; + pfd.events = POLLIN | POLLERR; - if (conn->cache.len > 0) { - /* - * The last invocation of fetch_read was interrupted by a - * signal after some data had been read from the socket. Copy - * the cached data into the supplied buffer before trying to - * read from the socket again. - */ - total = (conn->cache.len < len) ? conn->cache.len : len; - memcpy(buf, conn->cache.buf, total); - - conn->cache.len -= total; - conn->cache.pos += total; - len -= total; - buf += total; - } - - while (len > 0) { + for (;;) { /* * The socket is non-blocking. Instead of the canonical - * select() -> read(), we do the following: + * poll() -> read(), we do the following: * * 1) call read() or SSL_read(). - * 2) if an error occurred, return -1. - * 3) if we received data but we still expect more, - * update our counters and loop. + * 2) if we received some data, return it. + * 3) if an error occurred, return -1. * 4) if read() or SSL_read() signaled EOF, return. * 5) if we did not receive any data but we're not at EOF, - * call select(). + * call poll(). * * In the SSL case, this is necessary because if we * receive a close notification, we have to call @@ -1017,46 +976,34 @@ fetch_read(conn_t *conn, char *buf, size_t len) else #endif rlen = fetch_socket_read(conn->sd, buf, len); - if (rlen == 0) { + if (rlen > 0) { break; - } else if (rlen > 0) { - len -= rlen; - buf += rlen; - total += rlen; - continue; } else if (rlen == FETCH_READ_ERROR) { if (errno == EINTR) - fetch_cache_data(conn, start, total); + break; return (-1); } - // assert(rlen == FETCH_READ_WAIT); - FD_ZERO(&readfds); - while (!FD_ISSET(conn->sd, &readfds)) { - FD_SET(conn->sd, &readfds); - if (fetchTimeout > 0) { - gettimeofday(&now, NULL); - if (!timercmp(&timeout, &now, >)) { - errno = ETIMEDOUT; - fetch_syserr(); - return (-1); - } - timersub(&timeout, &now, &delta); - } - errno = 0; - if (select(conn->sd + 1, &readfds, NULL, NULL, - fetchTimeout > 0 ? &delta : NULL) < 0) { - if (errno == EINTR) { - if (fetchRestartCalls) - continue; - /* Save anything that was read. */ - fetch_cache_data(conn, start, total); - } + if (fetchTimeout > 0) { + gettimeofday(&now, NULL); + if (!timercmp(&timeout, &now, >)) { + errno = ETIMEDOUT; fetch_syserr(); return (-1); } + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000;; + } + errno = 0; + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { + if (errno == EINTR && fetchRestartCalls) + continue; + fetch_syserr(); + return (-1); } } - return (total); + return (rlen); } @@ -1130,20 +1077,21 @@ ssize_t fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) { struct timeval now, timeout, delta; - fd_set writefds; + struct pollfd pfd; ssize_t wlen, total; - int r; + int deltams, r; + memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { - FD_ZERO(&writefds); + pfd.fd = conn->sd; + pfd.events = POLLOUT | POLLERR; gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } total = 0; while (iovcnt > 0) { - while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { - FD_SET(conn->sd, &writefds); + while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); delta.tv_sec = timeout.tv_sec - now.tv_sec; delta.tv_usec = timeout.tv_usec - now.tv_usec; @@ -1156,9 +1104,9 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) fetch_syserr(); return (-1); } + deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000;; errno = 0; - r = select(conn->sd + 1, NULL, &writefds, NULL, &delta); - if (r == -1) { + if ((r = poll(&pfd, 1, deltams)) == -1) { if (errno == EINTR && fetchRestartCalls) continue; return (-1); @@ -1250,7 +1198,6 @@ fetch_close(conn_t *conn) } #endif ret = close(conn->sd); - free(conn->cache.buf); free(conn->buf); free(conn); return (ret); diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h index 1d543a616d00..ba45534b66b2 100644 --- a/lib/libfetch/common.h +++ b/lib/libfetch/common.h @@ -52,13 +52,6 @@ struct fetchconn { size_t bufsize; /* buffer size */ size_t buflen; /* length of buffer contents */ int err; /* last protocol reply code */ - struct { /* data cached after an interrupted - read */ - char *buf; - size_t size; - size_t pos; - size_t len; - } cache; #ifdef WITH_SSL SSL *ssl; /* SSL handle */ SSL_CTX *ssl_ctx; /* SSL context */ diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index 87535f001698..fc43a97fbdb1 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -208,6 +208,7 @@ static int http_fillbuf(struct httpio *io, size_t len) { ssize_t nbytes; + char ch; if (io->error) return (-1); @@ -249,10 +250,8 @@ http_fillbuf(struct httpio *io, size_t len) io->chunksize -= io->buflen; if (io->chunksize == 0) { - char endl[2]; - - if (fetch_read(io->conn, endl, 2) != 2 || - endl[0] != '\r' || endl[1] != '\n') + if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' || + fetch_read(io->conn, &ch, 1) != 1 || ch != '\n') return (-1); } @@ -268,31 +267,28 @@ static int http_readfn(void *v, char *buf, int len) { struct httpio *io = (struct httpio *)v; - int l, pos; + int rlen; if (io->error) return (-1); if (io->eof) return (0); - for (pos = 0; len > 0; pos += l, len -= l) { - /* empty buffer */ - if (!io->buf || io->bufpos == io->buflen) - if (http_fillbuf(io, len) < 1) - break; - l = io->buflen - io->bufpos; - if (len < l) - l = len; - memcpy(buf + pos, io->buf + io->bufpos, l); - io->bufpos += l; + /* empty buffer */ + if (!io->buf || io->bufpos == io->buflen) { + if (http_fillbuf(io, len) < 1) { + if (io->error == EINTR) + io->error = 0; + return (-1); + } } - if (!pos && io->error) { - if (io->error == EINTR) - io->error = 0; - return (-1); - } - return (pos); + rlen = io->buflen - io->bufpos; + if (len < rlen) + rlen = len; + memcpy(buf, io->buf + io->bufpos, rlen); + io->bufpos += rlen; + return (rlen); } /* -- cgit v1.3 From 9c1ca3a1dd4f199495e3892856e0ecfa16c1918d Mon Sep 17 00:00:00 2001 From: Dag-Erling Smørgrav Date: Wed, 29 Jan 2014 12:48:19 +0000 Subject: r261230 broke the cases where the amount of data to be read is not known in advance, or where the caller doesn't care and just keeps reading until it hits EOF. In fetch_read(): the socket is non-blocking, so read() will return 0 on EOF, and -1 (errno == EAGAIN) when the connection is still open but there is no data waiting. In the first case, we should immediately return 0. The EINTR case was also broken, although not in a way that matters. In fetch_writev(): use timersub() and timercmp() as in fetch_read(). In http_fillbuf(): set errno to a sensible value when an invalid chunk header is encountered. In http_readfn(): as in fetch_read(), a zero return from down the stack indicates EOF, not an error. Furthermore, when io->error is EINTR, clear it (but no errno) before returning so the caller can retry after dealing with the interrupt. MFC after: 3 days --- lib/libfetch/common.c | 23 ++++++++++------------- lib/libfetch/http.c | 10 ++++++---- 2 files changed, 16 insertions(+), 17 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index ab81b5821f73..94ebfa21edae 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -976,13 +976,13 @@ fetch_read(conn_t *conn, char *buf, size_t len) else #endif rlen = fetch_socket_read(conn->sd, buf, len); - if (rlen > 0) { + if (rlen >= 0) { break; } else if (rlen == FETCH_READ_ERROR) { - if (errno == EINTR) - break; + fetch_syserr(); return (-1); } + // assert(rlen == FETCH_READ_WAIT); if (fetchTimeout > 0) { gettimeofday(&now, NULL); if (!timercmp(&timeout, &now, >)) { @@ -1079,7 +1079,7 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) struct timeval now, timeout, delta; struct pollfd pfd; ssize_t wlen, total; - int deltams, r; + int deltams; memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { @@ -1093,20 +1093,17 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) while (iovcnt > 0) { while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); - delta.tv_sec = timeout.tv_sec - now.tv_sec; - delta.tv_usec = timeout.tv_usec - now.tv_usec; - if (delta.tv_usec < 0) { - delta.tv_usec += 1000000; - delta.tv_sec--; - } - if (delta.tv_sec < 0) { + if (!timercmp(&timeout, &now, >)) { errno = ETIMEDOUT; fetch_syserr(); return (-1); } - deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000;; + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000; errno = 0; - if ((r = poll(&pfd, 1, deltams)) == -1) { + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { if (errno == EINTR && fetchRestartCalls) continue; return (-1); diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index fc43a97fbdb1..676cbb76d960 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -204,7 +204,7 @@ http_growbuf(struct httpio *io, size_t len) /* * Fill the input buffer, do chunk decoding on the fly */ -static int +static ssize_t http_fillbuf(struct httpio *io, size_t len) { ssize_t nbytes; @@ -230,7 +230,7 @@ http_fillbuf(struct httpio *io, size_t len) if (io->chunksize == 0) { switch (http_new_chunk(io)) { case -1: - io->error = 1; + io->error = EPROTO; return (-1); case 0: io->eof = 1; @@ -276,10 +276,12 @@ http_readfn(void *v, char *buf, int len) /* empty buffer */ if (!io->buf || io->bufpos == io->buflen) { - if (http_fillbuf(io, len) < 1) { - if (io->error == EINTR) + if ((rlen = http_fillbuf(io, len)) < 0) { + if ((errno = io->error) == EINTR) io->error = 0; return (-1); + } else if (rlen == 0) { + return (0); } } -- cgit v1.3 From 4524013cd3ca6337f1c7ed57594ebb70fa4ab399 Mon Sep 17 00:00:00 2001 From: Dag-Erling Smørgrav Date: Thu, 30 Jan 2014 08:37:23 +0000 Subject: Bump copyright dates --- lib/libfetch/common.c | 2 +- lib/libfetch/common.h | 2 +- lib/libfetch/http.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 94ebfa21edae..5cc7e263b0c5 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 1998-2011 Dag-Erling Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * Copyright (c) 2013 Michael Gmelin * All rights reserved. * diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h index ba45534b66b2..875eef196e98 100644 --- a/lib/libfetch/common.h +++ b/lib/libfetch/common.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 1998-2011 Dag-Erling Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index 676cbb76d960..b2feedbcd0bf 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2000-2013 Dag-Erling Smørgrav + * Copyright (c) 2000-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without -- cgit v1.3 From b36853caf148377580a0a181357f5107d9d71355 Mon Sep 17 00:00:00 2001 From: Bryan Drewery Date: Tue, 11 Mar 2014 13:47:11 +0000 Subject: Support Last-Modified behind proxies which return UTC instead of GMT. The standard states that GMT must be used, but that UTC is equivalent. Still parse UTC as otherwise this causes problems for pkg(8). It will refetch the repository every time 'pkg update' or other remote operations are used behind these proxies. RFC2616: "All HTTP date/time stamps MUST be represented in Greenwich Mean Time (GMT), without exception. For the purposes of HTTP, GMT is exactly equal to UTC (Coordinated Universal Time)."" Approved by: bapt (mentor) Reviewed by: des, peter Sponsored by: EMC / Isilon Storage Division MFC after: 1 week --- lib/libfetch/http.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib/libfetch') diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index b2feedbcd0bf..cbbb8a8d351f 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -876,6 +876,12 @@ http_parse_mtime(const char *p, time_t *mtime) strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale)); setlocale(LC_TIME, "C"); r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); + /* + * Some proxies use UTC in response, but it should still be + * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP. + */ + if (r == NULL) + r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm); /* XXX should add support for date-2 and date-3 */ setlocale(LC_TIME, locale); if (r == NULL) -- cgit v1.3