diff options
Diffstat (limited to 'subversion/libsvn_fs_fs/revprops.c')
-rw-r--r-- | subversion/libsvn_fs_fs/revprops.c | 362 |
1 files changed, 251 insertions, 111 deletions
diff --git a/subversion/libsvn_fs_fs/revprops.c b/subversion/libsvn_fs_fs/revprops.c index dbb185beca79..6d41fd882d7e 100644 --- a/subversion/libsvn_fs_fs/revprops.c +++ b/subversion/libsvn_fs_fs/revprops.c @@ -25,9 +25,11 @@ #include "svn_pools.h" #include "svn_hash.h" #include "svn_dirent_uri.h" +#include "svn_sorts.h" #include "fs_fs.h" #include "revprops.h" +#include "temp_serializer.h" #include "util.h" #include "private/svn_subr_private.h" @@ -36,11 +38,6 @@ #include "svn_private_config.h" -/* Give writing processes 10 seconds to replace an existing revprop - file with a new one. After that time, we assume that the writing - process got aborted and that we have re-read revprops. */ -#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) - svn_error_t * svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, svn_fs_upgrade_notify_t notify_func, @@ -82,6 +79,7 @@ svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, shard, ffd->max_files_per_dir, (int)(0.9 * ffd->revprop_pack_size), compression_level, + ffd->flush_to_disk, cancel_func, cancel_baton, iterpool)); if (notify_func) @@ -144,9 +142,6 @@ typedef struct packed_revprops_t /* revision number to read (not necessarily the first in the pack) */ svn_revnum_t revision; - /* current revprop generation. Used when populating the revprop cache */ - apr_int64_t generation; - /* the actual revision properties */ apr_hash_t *properties; @@ -189,35 +184,84 @@ typedef struct packed_revprops_t /* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. * Also, put them into the revprop cache, if activated, for future use. - * Three more parameters are being used to update the revprop cache: FS is - * our file system, the revprops belong to REVISION and the global revprop - * GENERATION is used as well. * - * The returned hash will be allocated in POOL, SCRATCH_POOL is being used - * for temporary allocations. + * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is being + * used for temporary allocations. */ static svn_error_t * parse_revprop(apr_hash_t **properties, svn_fs_t *fs, svn_revnum_t revision, - apr_int64_t generation, svn_string_t *content, - apr_pool_t *pool, + apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); - *properties = apr_hash_make(pool); + *properties = apr_hash_make(result_pool); - SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool), + SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, + result_pool), apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", revision)); return SVN_NO_ERROR; } +void +svn_fs_fs__reset_revprop_cache(svn_fs_t *fs) +{ + fs_fs_data_t *ffd = fs->fsap_data; + ffd->revprop_prefix = 0; +} + +/* If FS has not a revprop cache prefix set, generate one. + * Always call this before accessing the revprop cache. + */ +static svn_error_t * +prepare_revprop_cache(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + if (!ffd->revprop_prefix) + SVN_ERR(svn_atomic__unique_counter(&ffd->revprop_prefix)); + + return SVN_NO_ERROR; +} + +/* Store the unparsed revprop hash CONTENT for REVISION in FS's revprop + * cache. If CACHED is not NULL, set *CACHED if there already is such + * an entry and skip the cache write in that case. Use SCRATCH_POOL for + * temporary allocations. */ +static svn_error_t * +cache_revprops(svn_boolean_t *is_cached, + svn_fs_t *fs, + svn_revnum_t revision, + svn_string_t *content, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + pair_cache_key_t key; + + /* Make sure prepare_revprop_cache() has been called. */ + SVN_ERR_ASSERT(ffd->revprop_prefix); + key.revision = revision; + key.second = ffd->revprop_prefix; + + if (is_cached) + { + SVN_ERR(svn_cache__has_key(is_cached, ffd->revprop_cache, &key, + scratch_pool)); + if (*is_cached) + return SVN_NO_ERROR; + } + + SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, content, scratch_pool)); + + return SVN_NO_ERROR; +} + /* Read the non-packed revprops for revision REV in FS, put them into the - * revprop cache if activated and return them in *PROPERTIES. GENERATION - * is the current revprop generation. + * revprop cache if PROPULATE_CACHE is set and return them in *PROPERTIES. * * If the data could not be read due to an otherwise recoverable error, * leave *PROPERTIES unchanged. No error will be returned in that case. @@ -228,7 +272,7 @@ static svn_error_t * read_non_packed_revprop(apr_hash_t **properties, svn_fs_t *fs, svn_revnum_t rev, - apr_int64_t generation, + svn_boolean_t populate_cache, apr_pool_t *pool) { svn_stringbuf_t *content = NULL; @@ -249,9 +293,13 @@ read_non_packed_revprop(apr_hash_t **properties, } if (content) - SVN_ERR(parse_revprop(properties, fs, rev, generation, - svn_stringbuf__morph_into_string(content), - pool, iterpool)); + { + svn_string_t *as_string = svn_stringbuf__morph_into_string(content); + SVN_ERR(parse_revprop(properties, fs, rev, as_string, pool, iterpool)); + + if (populate_cache) + SVN_ERR(cache_revprops(NULL, fs, rev, as_string, iterpool)); + } svn_pool_clear(iterpool); @@ -272,12 +320,13 @@ get_min_filename_len(packed_revprops_t *revprops) } /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST - * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. + * members. Use RESULT_POOL for allocating results and SCRATCH_POOL for + * temporaries. */ static svn_error_t * get_revprop_packname(svn_fs_t *fs, packed_revprops_t *revprops, - apr_pool_t *pool, + apr_pool_t *result_pool, apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; @@ -298,18 +347,20 @@ get_revprop_packname(svn_fs_t *fs, --rev_count; } - revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*)); + revprops->manifest = apr_array_make(result_pool, rev_count, + sizeof(const char*)); /* No line in the file can be less than this number of chars long. */ min_filename_len = get_min_filename_len(revprops); /* Read the content of the manifest file */ revprops->folder - = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool); + = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, + result_pool); manifest_file_path - = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool); - SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool)); + SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, result_pool)); /* There CONTENT must have a certain minimal size and there no * unterminated lines at the end of the file. Both guarantees also @@ -392,7 +443,8 @@ same_shard(svn_fs_t *fs, /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, * fill the START_REVISION member, and make PACKED_REVPROPS point to the * first serialized revprop. If READ_ALL is set, initialize the SIZES - * and OFFSETS members as well. + * and OFFSETS members as well. If POPULATE_CACHE is set, cache all + * revprops found in this pack. * * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as * well as the SERIALIZED_SIZE member. If revprop caching has been @@ -402,20 +454,25 @@ static svn_error_t * parse_packed_revprops(svn_fs_t *fs, packed_revprops_t *revprops, svn_boolean_t read_all, - apr_pool_t *pool, + svn_boolean_t populate_cache, + apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_stream_t *stream; apr_int64_t first_rev, count, i; - apr_off_t offset; + apr_size_t offset; const char *header_end; apr_pool_t *iterpool = svn_pool_create(scratch_pool); + /* Initial value for the "Leaking bucket" pattern. */ + int bucket = 4; + /* decompress (even if the data is only "stored", there is still a * length header to remove) */ svn_stringbuf_t *compressed = revprops->packed_revprops; - svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); - SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); + svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool); + SVN_ERR(svn__decompress_zlib(compressed->data, compressed->len, + uncompressed, APR_SIZE_MAX)); /* read first revision number and number of revisions in the pack */ stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); @@ -453,18 +510,21 @@ parse_packed_revprops(svn_fs_t *fs, offset = header_end - uncompressed->data + 2; - revprops->packed_revprops = svn_stringbuf_create_empty(pool); + revprops->packed_revprops = svn_stringbuf_create_empty(result_pool); revprops->packed_revprops->data = uncompressed->data + offset; revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); - revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); + revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize + - offset); /* STREAM still points to the first entry in the sizes list. */ revprops->start_revision = (svn_revnum_t)first_rev; if (read_all) { /* Init / construct REVPROPS members. */ - revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); - revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->sizes = apr_array_make(result_pool, (int)count, + sizeof(offset)); + revprops->offsets = apr_array_make(result_pool, (int)count, + sizeof(offset)); } /* Now parse, revision by revision, the size and content of each @@ -479,7 +539,7 @@ parse_packed_revprops(svn_fs_t *fs, /* read & check the serialized size */ SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream, iterpool)); - if (size + offset > (apr_int64_t)revprops->packed_revprops->len) + if (size > (apr_int64_t)revprops->packed_revprops->len - offset) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Packed revprop size exceeds pack file size")); @@ -489,21 +549,47 @@ parse_packed_revprops(svn_fs_t *fs, if (revision == revprops->revision) { + /* Parse (and possibly cache) the one revprop list we care about. */ SVN_ERR(parse_revprop(&revprops->properties, fs, revision, - revprops->generation, &serialized, - pool, iterpool)); + &serialized, result_pool, iterpool)); revprops->serialized_size = serialized.len; /* If we only wanted the revprops for REVISION then we are done. */ - if (!read_all) + if (!read_all && !populate_cache) break; } + if (populate_cache) + { + /* Adding all those revprops is expensive, in particular in a + * multi-threaded environment. There are situations where hit + * rates are low and revprops get evicted before re-using them. + * + * We try to detect thosse cases here. + * Only keep going while most (at least 2/3) aren't cached, yet. */ + svn_boolean_t already_cached; + SVN_ERR(cache_revprops(&already_cached, fs, revision, &serialized, + iterpool)); + + /* Stop populating the cache once we encountered too many entries + * already present relative to the numbers being added. */ + if (!already_cached) + { + ++bucket; + } + else + { + bucket -= 2; + if (bucket < 0) + populate_cache = FALSE; + } + } + if (read_all) { /* fill REVPROPS data structures */ - APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; - APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; + APR_ARRAY_PUSH(revprops->sizes, apr_size_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_size_t) = offset; } revprops->total_size += serialized.len; @@ -514,7 +600,7 @@ parse_packed_revprops(svn_fs_t *fs, } /* In filesystem FS, read the packed revprops for revision REV into - * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * *REVPROPS. Populate the revprop cache, if POPULATE_CACHE is set. * If you want to modify revprop contents / update REVPROPS, READ_ALL * must be set. Otherwise, only the properties of REV are being provided. * Allocate data in POOL. @@ -523,8 +609,8 @@ static svn_error_t * read_pack_revprop(packed_revprops_t **revprops, svn_fs_t *fs, svn_revnum_t rev, - apr_int64_t generation, svn_boolean_t read_all, + svn_boolean_t populate_cache, apr_pool_t *pool) { apr_pool_t *iterpool = svn_pool_create(pool); @@ -544,7 +630,6 @@ read_pack_revprop(packed_revprops_t **revprops, /* initialize the result data structure */ result = apr_pcalloc(pool, sizeof(*result)); result->revision = rev; - result->generation = generation; /* try to read the packed revprops. This may require retries if we have * concurrent writers. */ @@ -575,7 +660,8 @@ read_pack_revprop(packed_revprops_t **revprops, _("Failed to read revprop pack file for r%ld"), rev); /* parse it. RESULT will be complete afterwards. */ - err = parse_packed_revprops(fs, result, read_all, pool, iterpool); + err = parse_packed_revprops(fs, result, read_all, populate_cache, pool, + iterpool); svn_pool_destroy(iterpool); if (err) return svn_error_createf(SVN_ERR_FS_CORRUPT, err, @@ -594,16 +680,48 @@ svn_error_t * svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, - apr_pool_t *pool) + svn_boolean_t refresh, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = fs->fsap_data; - apr_int64_t generation = 0; + + /* Only populate the cache if we did not just cross a sync barrier. + * This is to eliminate overhead from code that always sets REFRESH. + * For callers that want caching, the caching kicks in on read "later". */ + svn_boolean_t populate_cache = !refresh; /* not found, yet */ *proplist_p = NULL; /* should they be available at all? */ - SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); + SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool)); + + if (refresh) + { + /* Previous cache contents is invalid now. */ + svn_fs_fs__reset_revprop_cache(fs); + } + else + { + /* Try cache lookup first. */ + svn_boolean_t is_cached; + pair_cache_key_t key; + + /* Auto-alloc prefix and construct the key. */ + SVN_ERR(prepare_revprop_cache(fs, scratch_pool)); + key.revision = rev; + key.second = ffd->revprop_prefix; + + /* The only way that this might error out is due to parser error. */ + SVN_ERR_W(svn_cache__get((void **) proplist_p, &is_cached, + ffd->revprop_cache, &key, result_pool), + apr_psprintf(scratch_pool, + "Failed to parse revprops for r%ld.", + rev)); + if (is_cached) + return SVN_NO_ERROR; + } /* if REV had not been packed when we began, try reading it from the * non-packed shard. If that fails, we will fall through to packed @@ -611,7 +729,7 @@ svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, if (!svn_fs_fs__is_packed_revprop(fs, rev)) { svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, - generation, pool); + populate_cache, result_pool); if (err) { if (!APR_STATUS_IS_ENOENT(err->apr_err) @@ -629,7 +747,8 @@ svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) { packed_revprops_t *revprops; - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, populate_cache, + result_pool)); *proplist_p = revprops->properties; } @@ -657,6 +776,7 @@ write_non_packed_revprop(const char **final_path, apr_hash_t *proplist, apr_pool_t *pool) { + fs_fs_data_t *ffd = fs->fsap_data; apr_file_t *file; svn_stream_t *stream; *final_path = svn_fs_fs__path_revprops(fs, rev, pool); @@ -671,7 +791,8 @@ write_non_packed_revprop(const char **final_path, SVN_ERR(svn_stream_close(stream)); /* Flush temporary file to disk and close it. */ - SVN_ERR(svn_io_file_flush_to_disk(file, pool)); + if (ffd->flush_to_disk) + SVN_ERR(svn_io_file_flush_to_disk(file, pool)); SVN_ERR(svn_io_file_close(file, pool)); return SVN_NO_ERROR; @@ -694,8 +815,10 @@ switch_to_new_revprop(svn_fs_t *fs, apr_array_header_t *files_to_delete, apr_pool_t *pool) { + fs_fs_data_t *ffd = fs->fsap_data; + SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference, - pool)); + ffd->flush_to_disk, pool)); /* Clean up temporary files, if necessary. */ if (files_to_delete) @@ -744,13 +867,13 @@ serialize_revprops_header(svn_stream_t *stream, * We only allocate a few bytes each iteration -- even with a * million iterations we would still be in good shape memory-wise. */ - apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); - SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", + apr_size_t size = APR_ARRAY_IDX(sizes, i, apr_size_t); + SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_SIZE_T_FMT "\n", size)); } /* the double newline char indicates the end of the header */ - SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); + SVN_ERR(svn_stream_puts(stream, "\n")); svn_pool_destroy(iterpool); return SVN_NO_ERROR; @@ -773,7 +896,7 @@ repack_revprops(svn_fs_t *fs, int end, int changed_index, svn_stringbuf_t *new_serialized, - apr_off_t new_total_size, + apr_size_t new_total_size, apr_file_t *file, apr_pool_t *pool) { @@ -802,10 +925,8 @@ repack_revprops(svn_fs_t *fs, } else { - apr_size_t size - = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); - apr_size_t offset - = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); + apr_size_t size = APR_ARRAY_IDX(revprops->sizes, i, apr_size_t); + apr_size_t offset = APR_ARRAY_IDX(revprops->offsets, i, apr_size_t); SVN_ERR(svn_stream_write(stream, revprops->packed_revprops->data + offset, @@ -816,16 +937,17 @@ repack_revprops(svn_fs_t *fs, SVN_ERR(svn_stream_close(stream)); /* compress / store the data */ - SVN_ERR(svn__compress(uncompressed, - compressed, - ffd->compress_packed_revprops - ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT - : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, + compressed, + ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE)); /* finally, write the content to the target file, flush and close it */ SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len, NULL, pool)); - SVN_ERR(svn_io_file_flush_to_disk(file, pool)); + if (ffd->flush_to_disk) + SVN_ERR(svn_io_file_flush_to_disk(file, pool)); SVN_ERR(svn_io_file_close(file, pool)); return SVN_NO_ERROR; @@ -848,7 +970,7 @@ repack_file_open(apr_file_t **file, { apr_int64_t tag; const char *tag_string; - svn_string_t *new_filename; + const char *new_filename; int i; int manifest_offset = (int)(revprops->start_revision - revprops->manifest_start); @@ -872,18 +994,18 @@ repack_file_open(apr_file_t **file, old_filename); SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); - new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, - revprops->start_revision + start, - ++tag); + new_filename = apr_psprintf(pool, "%ld.%" APR_INT64_T_FMT, + revprops->start_revision + start, + ++tag); /* update the manifest to point to the new file */ for (i = start; i < end; ++i) APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) - = new_filename->data; + = new_filename; /* open the file */ SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder, - new_filename->data, + new_filename, pool), APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); @@ -907,15 +1029,14 @@ write_packed_revprop(const char **final_path, { fs_fs_data_t *ffd = fs->fsap_data; packed_revprops_t *revprops; - apr_int64_t generation = 0; svn_stream_t *stream; apr_file_t *file; svn_stringbuf_t *serialized; - apr_off_t new_total_size; + apr_size_t new_total_size; int changed_index; /* read contents of the current pack file */ - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, FALSE, pool)); /* serialize the new revprops */ serialized = svn_stringbuf_create_empty(pool); @@ -929,7 +1050,7 @@ write_packed_revprop(const char **final_path, + serialized->len + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; - APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; + APR_ARRAY_IDX(revprops->sizes, changed_index, apr_size_t) = serialized->len; /* can we put the new data into the same pack as the before? */ if ( new_total_size < ffd->revprop_pack_size @@ -953,23 +1074,23 @@ write_packed_revprop(const char **final_path, int left = 0; int right = revprops->sizes->nelts - 1; - apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; - apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE; /* let left and right side grow such that their size difference * is minimal after each step. */ while (left <= right) - if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) - < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) + if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_size_t) + < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_size_t)) { - left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_size_t) + SVN_INT64_BUFFER_SIZE; ++left; } else { - right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) - + SVN_INT64_BUFFER_SIZE; + right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_size_t) + + SVN_INT64_BUFFER_SIZE; --right; } @@ -1025,17 +1146,16 @@ write_packed_revprop(const char **final_path, *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, svn_io_file_del_none, pool, pool)); - + stream = svn_stream_from_aprfile2(file, TRUE, pool); for (i = 0; i < revprops->manifest->nelts; ++i) { const char *filename = APR_ARRAY_IDX(revprops->manifest, i, const char*); - SVN_ERR(svn_io_file_write_full(file, filename, strlen(filename), - NULL, pool)); - SVN_ERR(svn_io_file_putc('\n', file, pool)); + SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); } - - SVN_ERR(svn_io_file_flush_to_disk(file, pool)); + SVN_ERR(svn_stream_close(stream)); + if (ffd->flush_to_disk) + SVN_ERR(svn_io_file_flush_to_disk(file, pool)); SVN_ERR(svn_io_file_close(file, pool)); } @@ -1069,6 +1189,9 @@ svn_fs_fs__set_revision_proplist(svn_fs_t *fs, SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, fs, rev, proplist, pool)); + /* Previous cache contents is invalid now. */ + svn_fs_fs__reset_revprop_cache(fs); + /* We use the rev file of this revision as the perms reference, * because when setting revprops for the first time, the revprop * file won't exist and therefore can't serve as its own reference. @@ -1167,6 +1290,7 @@ svn_fs_fs__copy_revprops(const char *pack_file_dir, apr_array_header_t *sizes, apr_size_t total_size, int compression_level, + svn_boolean_t flush_to_disk, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) @@ -1199,6 +1323,7 @@ svn_fs_fs__copy_revprops(const char *pack_file_dir, { const char *path; svn_stream_t *stream; + apr_file_t *file; svn_pool_clear(iterpool); @@ -1207,8 +1332,11 @@ svn_fs_fs__copy_revprops(const char *pack_file_dir, iterpool); /* Copy all the bits from the non-packed revprop file to the end of - * the pack file. */ - SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); + * the pack file. Use unbuffered apr_file_t since we're going to + * write using 16kb chunks. */ + SVN_ERR(svn_io_file_open(&file, path, APR_READ, APR_OS_DEFAULT, + iterpool)); + stream = svn_stream_from_aprfile2(file, FALSE, iterpool); SVN_ERR(svn_stream_copy3(stream, pack_stream, cancel_func, cancel_baton, iterpool)); } @@ -1217,12 +1345,14 @@ svn_fs_fs__copy_revprops(const char *pack_file_dir, SVN_ERR(svn_stream_close(pack_stream)); /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ - SVN_ERR(svn__compress(uncompressed, compressed, compression_level)); + SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, + compressed, compression_level)); /* write the pack file content to disk */ SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len, NULL, scratch_pool)); - SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool)); + if (flush_to_disk) + SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool)); SVN_ERR(svn_io_file_close(pack_file, scratch_pool)); svn_pool_destroy(iterpool); @@ -1235,8 +1365,9 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, const char *shard_path, apr_int64_t shard, int max_files_per_dir, - apr_off_t max_pack_size, + apr_int64_t max_pack_size, int compression_level, + svn_boolean_t flush_to_disk, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) @@ -1245,10 +1376,14 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, apr_file_t *manifest_file; svn_stream_t *manifest_stream; svn_revnum_t start_rev, end_rev, rev; - apr_off_t total_size; + apr_size_t total_size; apr_pool_t *iterpool = svn_pool_create(scratch_pool); apr_array_header_t *sizes; + /* Sanitize config file values. */ + apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1), + SVN_MAX_OBJECT_SIZE); + /* Some useful paths. */ manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, scratch_pool); @@ -1276,7 +1411,7 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, works. */ /* initialize the revprop size info */ - sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); + sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t)); total_size = 2 * SVN_INT64_BUFFER_SIZE; /* Iterate over the revisions in this shard, determine their size and @@ -1293,16 +1428,20 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, iterpool); SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); - /* if we already have started a pack file and this revprop cannot be - * appended to it, write the previous pack file. */ - if (sizes->nelts != 0 && - total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) + /* If we already have started a pack file and this revprop cannot be + * appended to it, write the previous pack file. Note this overflow + * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */ + if (sizes->nelts != 0 + && ( finfo.size > max_size + || total_size > max_size + || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size)) { SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, shard_path, start_rev, rev-1, - sizes, (apr_size_t)total_size, - compression_level, cancel_func, - cancel_baton, iterpool)); + sizes, total_size, + compression_level, flush_to_disk, + cancel_func, cancel_baton, + iterpool)); /* next pack file starts empty again */ apr_array_clear(sizes); @@ -1319,7 +1458,7 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, pack_filename)); /* add to list of files to put into the current pack file */ - APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; + APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size; total_size += SVN_INT64_BUFFER_SIZE + finfo.size; } @@ -1328,12 +1467,13 @@ svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, shard_path, start_rev, rev-1, sizes, (apr_size_t)total_size, - compression_level, cancel_func, - cancel_baton, iterpool)); + compression_level, flush_to_disk, + cancel_func, cancel_baton, iterpool)); /* flush the manifest file to disk and update permissions */ SVN_ERR(svn_stream_close(manifest_stream)); - SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool)); + if (flush_to_disk) + SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool)); SVN_ERR(svn_io_file_close(manifest_file, iterpool)); SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); @@ -1365,7 +1505,7 @@ svn_fs_fs__delete_revprops_shard(const char *shard_path, apr_psprintf(iterpool, "%d", i), iterpool); if (cancel_func) - SVN_ERR((*cancel_func)(cancel_baton)); + SVN_ERR(cancel_func(cancel_baton)); SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); } |