summaryrefslogtreecommitdiff
path: root/subversion/libsvn_fs_x/pack.c
diff options
context:
space:
mode:
Diffstat (limited to 'subversion/libsvn_fs_x/pack.c')
-rw-r--r--subversion/libsvn_fs_x/pack.c381
1 files changed, 167 insertions, 214 deletions
diff --git a/subversion/libsvn_fs_x/pack.c b/subversion/libsvn_fs_x/pack.c
index cdbb980293223..095b04ec82977 100644
--- a/subversion/libsvn_fs_x/pack.c
+++ b/subversion/libsvn_fs_x/pack.c
@@ -106,9 +106,6 @@ typedef struct path_order_t
/* when this change happened */
svn_revnum_t revision;
- /* this is a directory node */
- svn_boolean_t is_dir;
-
/* length of the expanded representation content */
apr_int64_t expanded_size;
@@ -219,7 +216,7 @@ typedef struct pack_context_t
* to NULL that we already processed. */
apr_array_header_t *reps;
- /* array of int, marking for each revision, the which offset their items
+ /* array of int, marking for each revision, at which offset their items
* begin in REPS. Will be filled in phase 2 and be cleared after
* each revision range. */
apr_array_header_t *rev_offsets;
@@ -247,6 +244,7 @@ initialize_pack_context(pack_context_t *context,
const char *shard_dir,
svn_revnum_t shard_rev,
int max_items,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -275,9 +273,9 @@ initialize_pack_context(pack_context_t *context,
context->pack_file_dir = pack_file_dir;
context->pack_file_path
= svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
- SVN_ERR(svn_io_file_open(&context->pack_file, context->pack_file_path,
- APR_WRITE | APR_BUFFERED | APR_BINARY | APR_EXCL
- | APR_CREATE, APR_OS_DEFAULT, pool));
+
+ SVN_ERR(svn_fs_x__batch_fsync_open_file(&context->pack_file, batch,
+ context->pack_file_path, pool));
/* Proto index files */
SVN_ERR(svn_fs_x__l2p_proto_index_open(
@@ -346,6 +344,7 @@ reset_pack_context(pack_context_t *context,
SVN_ERR(svn_io_file_trunc(context->reps_file, 0, scratch_pool));
svn_pool_clear(context->info_pool);
+ context->paths = svn_prefix_tree__create(context->info_pool);
return SVN_NO_ERROR;
}
@@ -382,8 +381,6 @@ close_pack_context(pack_context_t *context,
SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, scratch_pool));
SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, scratch_pool));
- SVN_ERR(svn_io_file_close(context->pack_file, scratch_pool));
-
return SVN_NO_ERROR;
}
@@ -395,7 +392,7 @@ static svn_error_t *
copy_file_data(pack_context_t *context,
apr_file_t *dest,
apr_file_t *source,
- apr_off_t size,
+ svn_filesize_t size,
apr_pool_t *scratch_pool)
{
/* most non-representation items will be small. Minimize the buffer
@@ -478,14 +475,16 @@ copy_item_to_temp(pack_context_t *context,
svn_fs_x__p2l_entry_t *entry,
apr_pool_t *scratch_pool)
{
+ apr_file_t *file;
svn_fs_x__p2l_entry_t *new_entry
= svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&new_entry->offset, temp_file,
+ scratch_pool));
APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry;
- SVN_ERR(copy_file_data(context, temp_file, rev_file->file, entry->size,
+ SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(copy_file_data(context, temp_file, file, entry->size,
scratch_pool));
return SVN_NO_ERROR;
@@ -567,17 +566,20 @@ copy_rep_to_temp(pack_context_t *context,
apr_pool_t *scratch_pool)
{
svn_fs_x__rep_header_t *rep_header;
+ svn_stream_t *stream;
+ apr_file_t *file;
apr_off_t source_offset = entry->offset;
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+ scratch_pool));
add_item_rep_mapping(context, entry);
/* read & parse the representation header */
- SVN_ERR(svn_fs_x__read_rep_header(&rep_header, rev_file->stream,
+ SVN_ERR(svn_fs_x__rev_file_stream(&stream, rev_file));
+ SVN_ERR(svn_fs_x__read_rep_header(&rep_header, stream,
scratch_pool, scratch_pool));
/* if the representation is a delta against some other rep, link the two */
@@ -594,10 +596,10 @@ copy_rep_to_temp(pack_context_t *context,
}
/* copy the whole rep (including header!) to our temp file */
- SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset,
- scratch_pool));
- SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file,
- entry->size, scratch_pool));
+ SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, source_offset));
+ SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(copy_file_data(context, context->reps_file, file, entry->size,
+ scratch_pool));
return SVN_NO_ERROR;
}
@@ -616,9 +618,6 @@ compare_dir_entries(const svn_sort__item_t *a,
const svn_fs_dirent_t *lhs = (const svn_fs_dirent_t *) a->value;
const svn_fs_dirent_t *rhs = (const svn_fs_dirent_t *) b->value;
- if (lhs->kind != rhs->kind)
- return lhs->kind == svn_node_dir ? -1 : 1;
-
return strcmp(lhs->name, rhs->name);
}
@@ -642,7 +641,7 @@ svn_fs_x__order_dir_entries(svn_fs_t *fs,
return result;
}
-/* Return a duplicate of the the ORIGINAL path and with special sub-strins
+/* Return a duplicate of the ORIGINAL path and with special sub-strings
* (e.g. "trunk") modified in such a way that have a lower lexicographic
* value than any other "normal" file name.
*/
@@ -687,25 +686,28 @@ copy_node_to_temp(pack_context_t *context,
path_order_t *path_order = apr_pcalloc(context->info_pool,
sizeof(*path_order));
svn_fs_x__noderev_t *noderev;
+ svn_stream_t *stream;
+ apr_file_t *file;
const char *sort_path;
apr_off_t source_offset = entry->offset;
/* read & parse noderev */
- SVN_ERR(svn_fs_x__read_noderev(&noderev, rev_file->stream, scratch_pool,
+ SVN_ERR(svn_fs_x__rev_file_stream(&stream, rev_file));
+ SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, scratch_pool,
scratch_pool));
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+ scratch_pool));
add_item_rep_mapping(context, entry);
/* copy the noderev to our temp file */
- SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset,
- scratch_pool));
- SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file,
- entry->size, scratch_pool));
+ SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, source_offset));
+ SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(copy_file_data(context, context->reps_file, file, entry->size,
+ scratch_pool));
/* if the node has a data representation, make that the node's "base".
* This will (often) cause the noderev to be placed right in front of
@@ -732,7 +734,6 @@ copy_node_to_temp(pack_context_t *context,
path_order->path = svn_prefix_string__create(context->paths, sort_path);
path_order->node_id = noderev->node_id;
path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set);
- path_order->is_dir = noderev->kind == svn_node_dir;
path_order->noderev_id = noderev->noderev_id;
APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order;
@@ -776,13 +777,8 @@ compare_path_order(const path_order_t * const * lhs_p,
const path_order_t * lhs = *lhs_p;
const path_order_t * rhs = *rhs_p;
- /* cluster all directories */
- int diff = rhs->is_dir - lhs->is_dir;
- if (diff)
- return diff;
-
/* lexicographic order on path and node (i.e. latest first) */
- diff = svn_prefix_string__compare(lhs->path, rhs->path);
+ int diff = svn_prefix_string__compare(lhs->path, rhs->path);
if (diff)
return diff;
@@ -826,7 +822,7 @@ sort_reps(pack_context_t *context)
/* Return the remaining unused bytes in the current block in CONTEXT's
* pack file.
*/
-static apr_ssize_t
+static apr_off_t
get_block_left(pack_context_t *context)
{
svn_fs_x__data_t *ffd = context->fs->fsap_data;
@@ -1234,7 +1230,7 @@ write_reps_containers(pack_context_t *context,
= apr_array_make(scratch_pool, 64, sizeof(svn_fs_x__id_t));
svn_fs_x__revision_file_t *file;
- SVN_ERR(svn_fs_x__wrap_temp_rev_file(&file, context->fs, temp_file,
+ SVN_ERR(svn_fs_x__rev_file_wrap_temp(&file, context->fs, temp_file,
scratch_pool));
/* copy all items in strict order */
@@ -1608,8 +1604,8 @@ write_changes_containers(pack_context_t *context,
* the container */
SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset,
iterpool));
- SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, scratch_pool,
- iterpool));
+ SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, INT_MAX,
+ scratch_pool, iterpool));
SVN_ERR(svn_fs_x__changes_append_list(&list_index, container, changes));
SVN_ERR_ASSERT(list_index == sub_items->nelts);
block_left -= estimated_size;
@@ -1723,18 +1719,19 @@ pack_range(pack_context_t *context,
{
apr_off_t offset = 0;
svn_fs_x__revision_file_t *rev_file;
+ svn_fs_x__index_info_t l2p_index_info;
/* Get the rev file dimensions (mainly index locations). */
- SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs,
- revision, revpool, iterpool));
- SVN_ERR(svn_fs_x__auto_read_footer(rev_file));
+ SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, revision,
+ revpool));
+ SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file));
/* store the indirect array index */
APR_ARRAY_PUSH(context->rev_offsets, int) = context->reps->nelts;
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < rev_file->l2p_offset)
+ while (offset < l2p_index_info.start)
{
/* read one cluster */
int i;
@@ -1758,10 +1755,9 @@ pack_range(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < rev_file->l2p_offset)
+ if (offset < l2p_index_info.start)
{
- SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &offset,
- iterpool));
+ SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset));
if (entry->type == SVN_FS_X__ITEM_TYPE_CHANGES)
SVN_ERR(copy_item_to_temp(context,
@@ -1843,21 +1839,19 @@ append_revision(pack_context_t *context,
apr_off_t offset = 0;
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
svn_fs_x__revision_file_t *rev_file;
- apr_finfo_t finfo;
-
- /* Get the size of the file. */
- const char *path = svn_dirent_join(context->shard_dir,
- apr_psprintf(iterpool, "%ld",
- context->start_rev),
- scratch_pool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, scratch_pool));
-
- /* Copy all the bits from the rev file to the end of the pack file. */
- SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs,
- context->start_rev, scratch_pool,
- iterpool));
- SVN_ERR(copy_file_data(context, context->pack_file, rev_file->file,
- finfo.size, iterpool));
+ apr_file_t *file;
+ svn_filesize_t revdata_size;
+
+ /* Copy all non-index contents the rev file to the end of the pack file. */
+ SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, context->start_rev,
+ scratch_pool));
+ SVN_ERR(svn_fs_x__rev_file_data_size(&revdata_size, rev_file));
+
+ SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
+ SVN_ERR(svn_io_file_aligned_seek(file, ffd->block_size, NULL, 0,
+ iterpool));
+ SVN_ERR(copy_file_data(context, context->pack_file, file, revdata_size,
+ iterpool));
/* mark the start of a new revision */
SVN_ERR(svn_fs_x__l2p_proto_index_add_revision(context->proto_l2p_index,
@@ -1865,7 +1859,7 @@ append_revision(pack_context_t *context,
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < finfo.size)
+ while (offset < revdata_size)
{
/* read one cluster */
int i;
@@ -1887,7 +1881,7 @@ append_revision(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < finfo.size)
+ if (offset < revdata_size)
{
/* there should be true containers */
SVN_ERR_ASSERT(entry->item_count == 1);
@@ -1906,7 +1900,7 @@ append_revision(pack_context_t *context,
}
svn_pool_destroy(iterpool);
- context->pack_offset += finfo.size;
+ context->pack_offset += revdata_size;
return SVN_NO_ERROR;
}
@@ -1917,6 +1911,7 @@ append_revision(pack_context_t *context,
* SHARD_DIR into the PACK_FILE_DIR, using SCRATCH_POOL for temporary
* allocations. Limit the extra memory consumption to MAX_MEM bytes.
* CANCEL_FUNC and CANCEL_BATON are what you think they are.
+ * Schedule necessary fsync calls in BATCH.
*/
static svn_error_t *
pack_log_addressed(svn_fs_t *fs,
@@ -1924,6 +1919,7 @@ pack_log_addressed(svn_fs_t *fs,
const char *shard_dir,
svn_revnum_t shard_rev,
apr_size_t max_mem,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -1950,7 +1946,7 @@ pack_log_addressed(svn_fs_t *fs,
/* set up a pack context */
SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir,
- shard_rev, max_items, cancel_func,
+ shard_rev, max_items, batch, cancel_func,
cancel_baton, scratch_pool));
/* phase 1: determine the size of the revisions to pack */
@@ -1960,8 +1956,10 @@ pack_log_addressed(svn_fs_t *fs,
/* pack revisions in ranges that don't exceed MAX_MEM */
for (i = 0; i < max_ids->nelts; ++i)
- if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items)
+ if ( APR_ARRAY_IDX(max_ids, i, apr_uint64_t)
+ <= (apr_uint64_t)max_items - item_count)
{
+ item_count += APR_ARRAY_IDX(max_ids, i, apr_uint64_t);
context.end_rev++;
}
else
@@ -2003,77 +2001,11 @@ pack_log_addressed(svn_fs_t *fs,
return SVN_NO_ERROR;
}
-/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
- Use SCRATCH_POOL for temporary allocations. */
-svn_error_t *
-svn_fs_x__get_packed_offset(apr_off_t *rev_offset,
- svn_fs_t *fs,
- svn_revnum_t rev,
- apr_pool_t *scratch_pool)
-{
- svn_fs_x__data_t *ffd = fs->fsap_data;
- svn_stream_t *manifest_stream;
- svn_boolean_t is_cached;
- svn_revnum_t shard;
- apr_int64_t shard_pos;
- apr_array_header_t *manifest;
- apr_pool_t *iterpool;
-
- shard = rev / ffd->max_files_per_dir;
-
- /* position of the shard within the manifest */
- shard_pos = rev % ffd->max_files_per_dir;
-
- /* fetch exactly that element into *rev_offset, if the manifest is found
- in the cache */
- SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached,
- ffd->packed_offset_cache, &shard,
- svn_fs_x__get_sharded_offset, &shard_pos,
- scratch_pool));
-
- if (is_cached)
- return SVN_NO_ERROR;
-
- /* Open the manifest file. */
- SVN_ERR(svn_stream_open_readonly(&manifest_stream,
- svn_fs_x__path_rev_packed(fs, rev, PATH_MANIFEST,
- scratch_pool),
- scratch_pool, scratch_pool));
-
- /* While we're here, let's just read the entire manifest file into an array,
- so we can cache the entire thing. */
- iterpool = svn_pool_create(scratch_pool);
- manifest = apr_array_make(scratch_pool, ffd->max_files_per_dir,
- sizeof(apr_off_t));
- while (1)
- {
- svn_boolean_t eof;
- apr_int64_t val;
-
- svn_pool_clear(iterpool);
- SVN_ERR(svn_fs_x__read_number_from_stream(&val, &eof, manifest_stream,
- iterpool));
- if (eof)
- break;
-
- APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
- }
- svn_pool_destroy(iterpool);
-
- *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir,
- apr_off_t);
-
- /* Close up shop and cache the array. */
- SVN_ERR(svn_stream_close(manifest_stream));
- return svn_cache__set(ffd->packed_offset_cache, &shard, manifest,
- scratch_pool);
-}
-
/* In filesystem FS, pack the revision SHARD containing exactly
* MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
* using SCRATCH_POOL for temporary allocations. Try to limit the amount of
* temporary memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON
- * are what you think they are.
+ * are what you think they are. Schedule necessary fsync calls in BATCH.
*
* If for some reason we detect a partial packing already performed, we
* remove the pack file and start again.
@@ -2087,6 +2019,7 @@ pack_rev_shard(svn_fs_t *fs,
apr_int64_t shard,
int max_files_per_dir,
apr_size_t max_mem,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -2103,10 +2036,11 @@ pack_rev_shard(svn_fs_t *fs,
/* Create the new directory and pack file. */
SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
+ SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, pack_file_dir, scratch_pool));
/* Index information files */
SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev,
- max_mem, cancel_func, cancel_baton,
+ max_mem, batch, cancel_func, cancel_baton,
scratch_pool));
SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, scratch_pool));
@@ -2115,11 +2049,10 @@ pack_rev_shard(svn_fs_t *fs,
return SVN_NO_ERROR;
}
-/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and
- * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using
- * SCRATCH_POOL temporary for allocations. REVPROPS_DIR will be NULL if
- * revprop packing is not supported. COMPRESSION_LEVEL and MAX_PACK_SIZE
- * will be ignored in that case.
+/* In the file system at FS_PATH, pack the SHARD in DIR containing exactly
+ * MAX_FILES_PER_DIR revisions, using SCRATCH_POOL temporary for allocations.
+ * COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that case.
+ * An attempt will be made to keep memory usage below MAX_MEM.
*
* CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
* NOTIFY_FUNC and NOTIFY_BATON.
@@ -2128,13 +2061,13 @@ pack_rev_shard(svn_fs_t *fs,
* remove the pack file and start again.
*/
static svn_error_t *
-pack_shard(const char *revs_dir,
- const char *revsprops_dir,
+pack_shard(const char *dir,
svn_fs_t *fs,
apr_int64_t shard,
int max_files_per_dir,
apr_off_t max_pack_size,
int compression_level,
+ apr_size_t max_mem,
svn_fs_pack_notify_t notify_func,
void *notify_baton,
svn_cancel_func_t cancel_func,
@@ -2142,49 +2075,42 @@ pack_shard(const char *revs_dir,
apr_pool_t *scratch_pool)
{
svn_fs_x__data_t *ffd = fs->fsap_data;
- const char *rev_shard_path, *rev_pack_file_dir;
- const char *revprops_shard_path, *revprops_pack_file_dir;
+ const char *shard_path, *pack_file_dir;
+ svn_fs_x__batch_fsync_t *batch;
/* Notify caller we're starting to pack this shard. */
if (notify_func)
SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
scratch_pool));
+ /* Perform all fsyncs through this instance. */
+ SVN_ERR(svn_fs_x__batch_fsync_create(&batch, ffd->flush_to_disk,
+ scratch_pool));
+
/* Some useful paths. */
- rev_pack_file_dir = svn_dirent_join(revs_dir,
+ pack_file_dir = svn_dirent_join(dir,
apr_psprintf(scratch_pool,
"%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
shard),
scratch_pool);
- rev_shard_path = svn_dirent_join(revs_dir,
+ shard_path = svn_dirent_join(dir,
apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
scratch_pool);
/* pack the revision content */
- SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path,
- shard, max_files_per_dir, DEFAULT_MAX_MEM,
+ SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path,
+ shard, max_files_per_dir, max_mem, batch,
cancel_func, cancel_baton, scratch_pool));
- /* if enabled, pack the revprops in an equivalent way */
- if (revsprops_dir)
- {
- revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
- apr_psprintf(scratch_pool,
- "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
- shard),
- scratch_pool);
- revprops_shard_path = svn_dirent_join(revsprops_dir,
- apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
- scratch_pool);
-
- SVN_ERR(svn_fs_x__pack_revprops_shard(revprops_pack_file_dir,
- revprops_shard_path,
- shard, max_files_per_dir,
- (int)(0.9 * max_pack_size),
- compression_level,
- cancel_func, cancel_baton,
- scratch_pool));
- }
+ /* pack the revprops in an equivalent way */
+ SVN_ERR(svn_fs_x__pack_revprops_shard(fs,
+ pack_file_dir,
+ shard_path,
+ shard, max_files_per_dir,
+ (int)(0.9 * max_pack_size),
+ compression_level, batch,
+ cancel_func, cancel_baton,
+ scratch_pool));
/* Update the min-unpacked-rev file to reflect our newly packed shard. */
SVN_ERR(svn_fs_x__write_min_unpacked_rev(fs,
@@ -2192,35 +2118,12 @@ pack_shard(const char *revs_dir,
scratch_pool));
ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir);
- /* Finally, remove the existing shard directories.
- * For revprops, clean up older obsolete shards as well as they might
- * have been left over from an interrupted FS upgrade. */
- SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
+ /* Ensure that packed file is written to disk.*/
+ SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool));
+
+ /* Finally, remove the existing shard directories. */
+ SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
cancel_func, cancel_baton, scratch_pool));
- if (revsprops_dir)
- {
- svn_node_kind_t kind = svn_node_dir;
- apr_int64_t to_cleanup = shard;
- do
- {
- SVN_ERR(svn_fs_x__delete_revprops_shard(revprops_shard_path,
- to_cleanup,
- max_files_per_dir,
- cancel_func, cancel_baton,
- scratch_pool));
-
- /* If the previous shard exists, clean it up as well.
- Don't try to clean up shard 0 as it we can't tell quickly
- whether it actually needs cleaning up. */
- revprops_shard_path = svn_dirent_join(revsprops_dir,
- apr_psprintf(scratch_pool,
- "%" APR_INT64_T_FMT,
- --to_cleanup),
- scratch_pool);
- SVN_ERR(svn_io_check_path(revprops_shard_path, &kind, scratch_pool));
- }
- while (kind == svn_node_dir && to_cleanup > 0);
- }
/* Notify caller we're starting to pack this shard. */
if (notify_func)
@@ -2230,9 +2133,38 @@ pack_shard(const char *revs_dir,
return SVN_NO_ERROR;
}
+/* Read the youngest rev and the first non-packed rev info for FS from disk.
+ Set *FULLY_PACKED when there is no completed unpacked shard.
+ Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+get_pack_status(svn_boolean_t *fully_packed,
+ svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ apr_int64_t completed_shards;
+ svn_revnum_t youngest;
+
+ SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, fs,
+ scratch_pool));
+
+ SVN_ERR(svn_fs_x__youngest_rev(&youngest, fs, scratch_pool));
+ completed_shards = (youngest + 1) / ffd->max_files_per_dir;
+
+ /* See if we've already completed all possible shards thus far. */
+ if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
+ *fully_packed = TRUE;
+ else
+ *fully_packed = FALSE;
+
+ return SVN_NO_ERROR;
+}
+
typedef struct pack_baton_t
{
svn_fs_t *fs;
+ apr_size_t max_mem;
svn_fs_pack_notify_t notify_func;
void *notify_baton;
svn_cancel_func_t cancel_func;
@@ -2262,25 +2194,25 @@ pack_body(void *baton,
svn_fs_x__data_t *ffd = pb->fs->fsap_data;
apr_int64_t completed_shards;
apr_int64_t i;
- svn_revnum_t youngest;
apr_pool_t *iterpool;
- const char *rev_data_path;
- const char *revprops_data_path = NULL;
-
- /* If we aren't using sharding, we can't do any packing, so quit. */
- SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs,
- scratch_pool));
+ const char *data_path;
+ svn_boolean_t fully_packed;
- SVN_ERR(svn_fs_x__youngest_rev(&youngest, pb->fs, scratch_pool));
- completed_shards = (youngest + 1) / ffd->max_files_per_dir;
+ /* Since another process might have already packed the repo,
+ we need to re-read the pack status. */
+ SVN_ERR(get_pack_status(&fully_packed, pb->fs, scratch_pool));
+ if (fully_packed)
+ {
+ if (pb->notify_func)
+ (*pb->notify_func)(pb->notify_baton,
+ ffd->min_unpacked_rev / ffd->max_files_per_dir,
+ svn_fs_pack_notify_noop, scratch_pool);
- /* See if we've already completed all possible shards thus far. */
- if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
- return SVN_NO_ERROR;
+ return SVN_NO_ERROR;
+ }
- rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool);
- revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR,
- scratch_pool);
+ completed_shards = (ffd->youngest_rev_cache + 1) / ffd->max_files_per_dir;
+ data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool);
iterpool = svn_pool_create(scratch_pool);
for (i = ffd->min_unpacked_rev / ffd->max_files_per_dir;
@@ -2292,12 +2224,13 @@ pack_body(void *baton,
if (pb->cancel_func)
SVN_ERR(pb->cancel_func(pb->cancel_baton));
- SVN_ERR(pack_shard(rev_data_path, revprops_data_path,
+ SVN_ERR(pack_shard(data_path,
pb->fs, i, ffd->max_files_per_dir,
ffd->revprop_pack_size,
ffd->compress_packed_revprops
? SVN__COMPRESSION_ZLIB_DEFAULT
: SVN__COMPRESSION_NONE,
+ pb->max_mem,
pb->notify_func, pb->notify_baton,
pb->cancel_func, pb->cancel_baton, iterpool));
}
@@ -2308,6 +2241,7 @@ pack_body(void *baton,
svn_error_t *
svn_fs_x__pack(svn_fs_t *fs,
+ apr_size_t max_mem,
svn_fs_pack_notify_t notify_func,
void *notify_baton,
svn_cancel_func_t cancel_func,
@@ -2315,10 +2249,29 @@ svn_fs_x__pack(svn_fs_t *fs,
apr_pool_t *scratch_pool)
{
pack_baton_t pb = { 0 };
+ svn_boolean_t fully_packed;
+
+ /* Is there we even anything to do?. */
+ SVN_ERR(get_pack_status(&fully_packed, fs, scratch_pool));
+ if (fully_packed)
+ {
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+
+ if (notify_func)
+ (*notify_func)(notify_baton,
+ ffd->min_unpacked_rev / ffd->max_files_per_dir,
+ svn_fs_pack_notify_noop, scratch_pool);
+
+ return SVN_NO_ERROR;
+ }
+
+ /* Lock the repo and start the pack process. */
pb.fs = fs;
pb.notify_func = notify_func;
pb.notify_baton = notify_baton;
pb.cancel_func = cancel_func;
pb.cancel_baton = cancel_baton;
+ pb.max_mem = max_mem ? max_mem : DEFAULT_MAX_MEM;
+
return svn_fs_x__with_pack_lock(fs, pack_body, &pb, scratch_pool);
}