6 files changed, 469 insertions, 13 deletions

diff --git a/contrib/csup/Makefile b/contrib/csup/Makefile
index 388839672759..d82a0f1774d3 100644
--- a/contrib/csup/Makefile
+++ b/contrib/csup/Makefile
@@ -10,7 +10,7 @@ PROG=	csup
 SRCS=	attrstack.c config.c detailer.c diff.c fattr.c fixups.c fnmatch.c \
 	globtree.c idcache.c keyword.c lister.c main.c misc.c mux.c parse.y \
 	pathcomp.c proto.c status.c stream.c threads.c token.l updater.c \
-	rcsfile.c rcsparse.c lex.rcs.c
+	rcsfile.c rcsparse.c lex.rcs.c rsyncfile.c
 
 CFLAGS+=	-I. -I${.CURDIR} -g -pthread -DHAVE_FFLAGS -DNDEBUG
 WARNS?=		1
diff --git a/contrib/csup/config.c b/contrib/csup/config.c
index 3c5b83e05370..393a3db58293 100644
--- a/contrib/csup/config.c
+++ b/contrib/csup/config.c
@@ -133,8 +133,6 @@ config_init(const char *file, struct coll *override, int overridemask)
 			coll->co_options &= ~CO_CHECKRCS;
 		/* In recent versions, we always try to set the file modes. */
 		coll->co_options |= CO_SETMODE;
-		/* XXX We don't support the rsync updating algorithm yet. */
-		coll->co_options |= CO_NORSYNC;
 		error = config_parse_refusefiles(coll);
 		if (error)
 			goto bad;
diff --git a/contrib/csup/detailer.c b/contrib/csup/detailer.c
index a07fe8b84835..dc16d1056d50 100644
--- a/contrib/csup/detailer.c
+++ b/contrib/csup/detailer.c
@@ -43,6 +43,7 @@
 #include "mux.h"
 #include "proto.h"
 #include "rcsfile.h"
+#include "rsyncfile.h"
 #include "status.h"
 #include "stream.h"
 
@@ -67,6 +68,7 @@ static int	detailer_dofile_co(struct detailer *, struct coll *,
 static int	detailer_dofile_rcs(struct detailer *, struct coll *, 
 		    char *, char *);
 static int	detailer_dofile_regular(struct detailer *, char *, char *);
+static int	detailer_dofile_rsync(struct detailer *, char *, char *);
 static int	detailer_checkrcsattr(struct detailer *, struct coll *, char *,
 		    struct fattr *, int);
 int		detailer_send_details(struct detailer *, struct coll *, char *,
@@ -106,7 +108,6 @@ detailer(void *arg)
 			xasprintf(&args->errmsg, "Detailer failed: "
 			    "Network read failure: %s", strerror(errno));
 		}
-		lprintf(-1, "Error is '%s'\n", args->errmsg);
 		args->status = STATUS_TRANSIENTFAILURE;
 		break;
 	case DETAILER_ERR_WRITE:
@@ -343,7 +344,6 @@ detailer_coll(struct detailer *d, struct coll *coll, struct status *st)
 			lprintf(-1, "Server warning: %s\n", msg);
 			break;
 		default:
-			lprintf(-1, "Line: %s, cmd %s\n", line, cmd);
 			return (DETAILER_ERR_PROTO);
 		}
 		stream_flush(wr);
@@ -390,6 +390,33 @@ detailer_dofile_regular(struct detailer *d, char *name, char *path)
 }
 
 /*
+ * Tell the server to update a file with the rsync algorithm.
+ */
+static int
+detailer_dofile_rsync(struct detailer *d, char *name, char *path)
+{
+	struct stream *wr;
+	struct rsyncfile *rf;
+
+	rf = rsync_open(path, 0, 1);
+	if (rf == NULL) {
+		/* Fallback if we fail in opening it. */
+		proto_printf(wr, "A %s\n", name);
+		return (0);
+	}
+	wr = d->wr;
+	proto_printf(wr, "r %s %z %z\n", name, rsync_filesize(rf),
+	    rsync_blocksize(rf));
+	/* Detail the blocks. */
+	while (rsync_nextblock(rf) != 0) {
+		proto_printf(wr, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf));
+		lprintf(-1, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf));
+	}
+	proto_printf(wr, ".\n");
+	rsync_close(rf);
+}
+
+/*
  * Tell the server to update an RCS file that we have, or send it if we don't.
  */
 static int
@@ -572,6 +599,9 @@ detailer_send_details(struct detailer *d, struct coll *coll, char *name,
 	} else if (fattr_type(fa) == FT_FILE) {
 		if (isrcs(name, &len) && !(coll->co_options & CO_NORCS)) {
 			detailer_dofile_rcs(d, coll, name, path);
+		} else if (!(coll->co_options & CO_NORSYNC) ||
+		    !globtree_test(coll->co_norsync, name)) {
+			detailer_dofile_rsync(d, name, path);
 		} else {
 			detailer_dofile_regular(d, name, path);
 		}
diff --git a/contrib/csup/rsyncfile.c b/contrib/csup/rsyncfile.c
new file mode 100644
index 000000000000..a61b1cba5a28
--- /dev/null
+++ b/contrib/csup/rsyncfile.c
@@ -0,0 +1,228 @@
+/*-
+ * Copyright (c) 2008, Ulf Lilleengen <lulf@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "misc.h"
+#include "fattr.h"
+#include "rsyncfile.h"
+
+#define MINBLOCKSIZE 1024
+#define MAXBLOCKSIZE (16 * 1024)
+#define RECEIVEBUFFERSIZE (15 * 1024)
+#define BLOCKINFOSIZE 26
+#define SEARCHREGION 10
+#define MAXBLOCKS (RECEIVEBUFFERSIZE / BLOCKINFOSIZE)
+
+#define CHAR_OFFSET 3
+#define RSUM_SIZE 9
+
+struct rsyncfile {
+	char *start;
+	char *buf;
+	char *end;
+	size_t blocksize;
+	size_t fsize;
+	struct fattr *fa;
+	int fd;
+
+	char *blockptr;
+	int blocknum;
+	char blockmd5[MD5_DIGEST_SIZE];
+	char rsumstr[RSUM_SIZE];
+	uint32_t rsum;
+};
+
+static size_t	rsync_chooseblocksize(size_t);
+static uint32_t	rsync_rollsum(uint8_t *, size_t);
+
+/* Open a file and initialize variable for rsync operation. */
+struct rsyncfile *
+rsync_open(char *path, size_t blocksize, int read)
+{
+	struct rsyncfile *rf;
+	struct stat st;
+	int error;
+
+	rf = malloc(sizeof(*rf));
+	if (rf == NULL)
+		return (NULL);
+	error = stat(path, &st);
+	if (error) {
+		free(rf);
+		return (NULL);
+	}
+	rf->fsize = st.st_size;
+	rf->fa = fattr_fromstat(&st);
+
+	rf->fd = open(path, read ? O_RDONLY : O_RDWR);
+	if (rf->fd < 0) {
+		free(rf);
+		return (NULL);
+	}
+	rf->buf = mmap(0, rf->fsize, PROT_READ, MAP_SHARED, rf->fd, 0);
+	if (rf->buf == MAP_FAILED) {
+		free(rf);
+		return (NULL);
+	}
+	rf->start = rf->buf;
+	rf->end = rf->buf + rf->fsize;
+	rf->blocksize = (blocksize == 0 ? rsync_chooseblocksize(rf->fsize) :
+	    blocksize);
+	rf->blockptr = rf->buf;
+	rf->blocknum = 0;
+	return (rf);
+}
+
+/* Close and free all resources related to an rsync file transfer. */
+int
+rsync_close(struct rsyncfile *rf)
+{
+	int error;
+
+	error = munmap(rf->buf, rf->fsize);
+	if (error)
+		return (error);
+	close(rf->fd);
+	free(rf);
+}
+
+/*
+ * Choose the most appropriate block size for an rsync transfer. Modeled
+ * algorithm after cvsup.
+ */
+static size_t
+rsync_chooseblocksize(size_t fsize)
+{
+	size_t bestrem, blocksize, bs, hisearch, losearch, rem;
+
+	blocksize = fsize / MAXBLOCKS;
+	losearch = blocksize - SEARCHREGION;
+	hisearch = blocksize + SEARCHREGION;
+
+	if (losearch < MINBLOCKSIZE) {
+		losearch = MINBLOCKSIZE;
+		hisearch = losearch + (2 * SEARCHREGION);
+	} else if (hisearch > MAXBLOCKSIZE) {
+		hisearch = MAXBLOCKSIZE;
+		losearch = hisearch - (2 * SEARCHREGION);
+	}
+
+	bestrem = MAXBLOCKSIZE;
+	for (bs = losearch; bs <= hisearch;) {
+		rem = fsize % bs;
+		if (rem < bestrem) {
+			bestrem = rem;
+			blocksize = bs;
+		}
+	}
+	return (bestrem);
+}
+
+/* Get the next rsync block of a file. */
+int
+rsync_nextblock(struct rsyncfile *rf)
+{
+	uint32_t rolling;
+	char *ptr;
+	MD5_CTX ctx;
+	size_t blocksize, i;
+
+	if (rf->blockptr >= rf->end)
+		return (0);
+	blocksize = min((rf->end - rf->blockptr), rf->blocksize);
+	/* Calculate MD5 of the block. */
+	MD5_Init(&ctx);
+	MD5_Update(&ctx, rf->blockptr, blocksize);
+	MD5_End(rf->blockmd5, &ctx);
+
+	rf->rsum = rsync_rollsum(rf->blockptr, blocksize);
+	snprintf(rf->rsumstr, RSUM_SIZE, "%x", rf->rsum);
+	rf->blocknum++;
+	rf->blockptr += blocksize;
+	return (1);
+}
+
+/* Get the rolling checksum of a file. */
+static uint32_t
+rsync_rollsum(uint8_t *buf, size_t len)
+{
+	uint32_t a, b;
+	uint8_t *ptr, *limit;
+
+	a = b = 0;
+	ptr = buf;
+	limit = buf + len;
+
+	while (ptr < limit) {
+		a += *ptr + CHAR_OFFSET;
+		b += a;
+		ptr++;
+	}
+	return ((b << 16) | a);
+}
+
+/* Get running sum so far. */
+char *
+rsync_rsum(struct rsyncfile *rf)
+{
+
+	return (rf->rsumstr);
+}
+
+/* Get MD5 of current block. */
+char *
+rsync_blockmd5(struct rsyncfile *rf)
+{
+
+	return (rf->blockmd5);
+}
+
+/* Accessor for blocksize. */
+size_t
+rsync_blocksize(struct rsyncfile *rf)
+{
+
+	return (rf->blocksize);
+}
+
+/* Accessor for filesize. */
+size_t
+rsync_filesize(struct rsyncfile *rf)
+{
+
+	return (rf->fsize);
+}
diff --git a/contrib/csup/rsyncfile.h b/contrib/csup/rsyncfile.h
new file mode 100644
index 000000000000..bfe1ad18e9ba
--- /dev/null
+++ b/contrib/csup/rsyncfile.h
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2008, Ulf Lilleengen <lulf@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _RSYNCFILE_H_
+#define _RSYNCFILE_H_
+
+struct rsyncfile;
+
+struct rsyncfile *rsync_open(char *, size_t, int);
+int		  rsync_nextblock(struct rsyncfile *);
+char		 *rsync_rsum(struct rsyncfile *);
+char		 *rsync_blockmd5(struct rsyncfile *);
+int		  rsync_close(struct rsyncfile *);
+size_t		  rsync_blocksize(struct rsyncfile *);
+size_t		  rsync_filesize(struct rsyncfile *);
+
+#endif /* !_RSYNCFILE_H_ */
diff --git a/contrib/csup/updater.c b/contrib/csup/updater.c
index 9d99ef7bdf8b..69526953c51e 100644
--- a/contrib/csup/updater.c
+++ b/contrib/csup/updater.c
@@ -115,6 +115,8 @@ static int	 updater_rcsedit(struct updater *, struct file_update *, char *,
 		     char *);
 int		 updater_append_file(struct updater *, struct file_update *,
 		     off_t);
+static int	 updater_rsync(struct updater *, struct file_update *, size_t);
+static int	 updater_read_checkout(struct stream *, struct stream *);
 
 static struct file_update *
 fup_new(struct coll *coll, struct status *st)
@@ -345,7 +347,7 @@ updater_docoll(struct updater *up, struct file_update *fup, int isfixups)
 	struct coll *coll;
 	struct statusrec srbuf, *sr;
 	struct fattr *rcsattr, *tmp;
-	char *cmd, *line, *msg, *attr;
+	char *cmd, *blocksize, *line, *msg, *attr;
 	char *name, *tag, *date, *revdate;
 	char *expand, *wantmd5, *revnum;
 	char *optstr, *rcsopt, *pos;
@@ -601,7 +603,30 @@ updater_docoll(struct updater *up, struct file_update *fup, int isfixups)
 			if (error)
 				return (error);
 			break;
-		case 'r': /* XXX: rsync support. */
+		case 'r':
+			name = proto_get_ascii(&line);
+			attr = proto_get_ascii(&line);
+			blocksize = proto_get_ascii(&line);
+			wantmd5 = proto_get_ascii(&line);
+			if (name == NULL || attr == NULL || blocksize == NULL ||
+			    wantmd5 == NULL) {
+				return (UPDATER_ERR_PROTO);
+			}
+			error = fup_prepare(fup, name, 0);
+			if (error)
+				return (UPDATER_ERR_PROTO);
+			sr->sr_type = SR_FILELIVE;
+			fup->wantmd5 = xstrdup(wantmd5);
+			fup->temppath = tempname(fup->destpath);
+			sr = &fup->srbuf;
+			sr->sr_file = xstrdup(name);
+			sr->sr_serverattr = fattr_decode(attr);
+			if (sr->sr_serverattr == NULL)
+				return (UPDATER_ERR_PROTO);
+			error = updater_rsync(up, fup, strtol(blocksize, NULL,
+			    10));
+			if (error)
+				return (error);
 			break;
 
 		case 'I':
@@ -1457,8 +1482,6 @@ updater_addfile(struct updater *up, struct file_update *fup, char *attr)
 		remains -= nread;
 		stream_write(to, buf, nread);
 	} while (remains > 0);
-	stream_flush(to);
-	stream_filter_stop(to);
 	stream_close(to);
 	line = stream_getln(up->rd, NULL);
 	if (line == NULL)
@@ -1749,8 +1772,6 @@ updater_rcsedit(struct updater *up, struct file_update *fup, char *name,
 	}
 	stream_filter_start(dest, STREAM_FILTER_MD5RCS, md5);
 	error = rcsfile_write(rf, dest);
-	stream_flush(dest);
-	stream_filter_stop(dest);
 	stream_close(dest);
 	rcsfile_free(rf);
 	if (error)
@@ -1927,8 +1948,6 @@ updater_append_file(struct updater *up, struct file_update *fup, off_t pos)
 		bytes -= nread;
 		stream_write(to, buf, nread);
 	} while (bytes > 0);
-	stream_flush(to);
-	stream_filter_stop(to);
 	stream_close(to);
 
 	line = stream_getln(up->rd, NULL);
@@ -1959,3 +1978,142 @@ updater_append_file(struct updater *up, struct file_update *fup, off_t pos)
 		return (error);
 	return (0);
 }
+
+/*
+ * Read file data from stream of checkout commands, and write it to the
+ * destination.
+ */
+static int
+updater_read_checkout(struct stream *src, struct stream *dest)
+{
+	char *line;
+	size_t size;
+	ssize_t nbytes;
+	int error, first;
+
+	first = 1;
+	line = stream_getln(src, &size);
+	while (line != NULL) {
+		if (line[size - 1] == '\n')
+			size--;
+		if ((size == 1 && *line == '.') ||
+		    (size == 2 && strncmp(line, ".+", 2) == 0))
+			break;
+		if (size >= 2 && strncmp(line, "..", 2) == 0) {
+			size--;
+			line++;
+		}
+		if (!first) {
+			nbytes = stream_write(dest, "\n", 1);
+			if (nbytes == -1)
+				return (UPDATER_ERR_MSG);
+		}
+		nbytes = stream_write(dest, line, size);
+		if (nbytes == -1)
+			return (UPDATER_ERR_MSG);
+		line = stream_getln(src, &size);
+		first = 0;
+	}
+	if (line == NULL)
+		return (UPDATER_ERR_READ);
+	if (size == 1 && *line == '.') {
+		nbytes = stream_write(dest, "\n", 1);
+		if (nbytes == -1)
+			return (UPDATER_ERR_MSG);
+	}
+	return (0);
+}
+
+/* Update file using the rsync protocol. */
+static int
+updater_rsync(struct updater *up, struct file_update *fup, size_t blocksize)
+{
+	struct statusrec *sr;
+	struct coll *coll;
+	struct stream *to;
+	char md5[MD5_DIGEST_SIZE];
+	char *buf, *line;
+	int error, orig;
+	size_t size, blocknum, blockstart, blockcount;
+	ssize_t nbytes;
+
+	sr = &fup->srbuf;
+
+	lprintf(1, " Rsync %s\n", fup->coname);
+	/* First open all files that we are going to work on. */
+	to = stream_open_file(fup->temppath, O_WRONLY | O_CREAT | O_TRUNC,
+	    0600);
+	if (to == NULL) {
+		xasprintf(&up->errmsg, "%s: Cannot create: %s",
+		    fup->temppath, strerror(errno));
+		return (UPDATER_ERR_MSG);
+	}
+	orig = open(fup->destpath, O_RDONLY);
+	if (orig < 0) {
+		xasprintf(&up->errmsg, "%s: Cannot open: %s",
+		    fup->destpath, strerror(errno));
+		return (UPDATER_ERR_MSG);
+	}
+	stream_filter_start(to, STREAM_FILTER_MD5, md5);
+
+	error = updater_read_checkout(up->rd, to);
+	if (error) {
+		xasprintf(&up->errmsg, "%s: Cannot write: %s", fup->temppath,
+		    strerror(errno));
+		return (error);
+	}
+
+	/* Buffer must contain blocksize bytes. */
+	buf = xmalloc(blocksize);
+	/* Done with the initial text, read and write chunks. */
+	line = stream_getln(up->rd, NULL);
+	while (line != NULL) {
+		if (strcmp(line, ".") == 0)
+			break;
+		error = UPDATER_ERR_PROTO;
+		if (proto_get_sizet(&line, &blockstart, 10) != 0)
+			goto bad;
+		if (proto_get_sizet(&line, &blockcount, 10) != 0)
+			goto bad;
+		/* Read blocks from original file. */
+		lseek(orig, SEEK_SET, (blocksize * blockstart));
+		blocknum = 0;
+		error = UPDATER_ERR_MSG;
+		for (blocknum = 0; blocknum < blockcount; blocknum++) {
+			nbytes = read(orig, buf, blocksize);
+			if (nbytes < 0) {
+				xasprintf(&up->errmsg, "%s: Cannot read: %s",
+				    fup->destpath, strerror(errno));
+				goto bad;
+			}
+			nbytes = stream_write(to, buf, nbytes);
+			if (nbytes == -1) {
+				xasprintf(&up->errmsg, "%s: Cannot write: %s",
+				    fup->temppath, strerror(errno));
+				goto bad;
+			}
+		}
+		/* Get the remaining text from the server. */
+		error = updater_read_checkout(up->rd, to);
+		if (error) {
+			xasprintf(&up->errmsg, "%s: Cannot write: %s",
+			    fup->temppath, strerror(errno));
+			goto bad;
+		}
+		line = stream_getln(up->rd, NULL);
+	}
+	stream_close(to);
+	close(orig);
+
+	sr->sr_clientattr = fattr_frompath(fup->destpath, FATTR_NOFOLLOW);
+	if (sr->sr_clientattr == NULL)
+		return (UPDATER_ERR_PROTO);
+	fattr_override(sr->sr_clientattr, sr->sr_serverattr,
+	    FA_MODTIME | FA_MASK);
+
+	error = updater_updatefile(up, fup, md5, 0);
+	fup->wantmd5 = NULL;	/* So that it doesn't get freed. */
+bad:
+	free(buf);
+	return (error);
+}