1 files changed, 179 insertions, 27 deletions

diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index da3ed3f1cd11..ccdc3b9eb5c9 100644
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -17,11 +17,42 @@
 /// the --threads=NUM command line option.
 static uint32_t threads_max = 1;
 
+/// True when the number of threads is automatically determined based
+/// on the available hardware threads.
+static bool threads_are_automatic = false;
+
+/// If true, then try to use multi-threaded mode (if memlimit allows)
+/// even if only one thread was requested explicitly (-T+1).
+static bool use_mt_mode_with_one_thread = false;
+
 /// Memory usage limit for compression
-static uint64_t memlimit_compress;
+static uint64_t memlimit_compress = 0;
 
 /// Memory usage limit for decompression
-static uint64_t memlimit_decompress;
+static uint64_t memlimit_decompress = 0;
+
+/// Default memory usage for multithreaded modes:
+///
+///   - Default value for --memlimit-compress when automatic number of threads
+///     is used. However, if the limit wouldn't allow even one thread then
+///     the limit is ignored in coder.c and one thread will be used anyway.
+///     This mess is a compromise: we wish to prevent -T0 from using too
+///     many threads but we also don't want xz to give an error due to
+///     a memlimit that the user didn't explicitly set.
+///
+///   - Default value for --memlimit-mt-decompress
+///
+/// This value is caluclated in hardware_init() and cannot be changed later.
+static uint64_t memlimit_mt_default;
+
+/// Memory usage limit for multithreaded decompression. This is a soft limit:
+/// if reducing the number of threads to one isn't enough to keep memory
+/// usage below this limit, then one thread is used and this limit is ignored.
+/// memlimit_decompress is still obeyed.
+///
+/// This can be set with --memlimit-mt-decompress. The default value for
+/// this is memlimit_mt_default.
+static uint64_t memlimit_mtdec;
 
 /// Total amount of physical RAM
 static uint64_t total_ram;
@@ -30,8 +61,17 @@ static uint64_t total_ram;
 extern void
 hardware_threads_set(uint32_t n)
 {
+	// Reset these to false first and set them to true when appropriate.
+	threads_are_automatic = false;
+	use_mt_mode_with_one_thread = false;
+
 	if (n == 0) {
 		// Automatic number of threads was requested.
+		// If there is only one hardware thread, multi-threaded
+		// mode will still be used if memory limit allows.
+		threads_are_automatic = true;
+		use_mt_mode_with_one_thread = true;
+
 		// If threading support was enabled at build time,
 		// use the number of available CPU cores. Otherwise
 		// use one thread since disabling threading support
@@ -43,6 +83,9 @@ hardware_threads_set(uint32_t n)
 #else
 		threads_max = 1;
 #endif
+	} else if (n == UINT32_MAX) {
+		use_mt_mode_with_one_thread = true;
+		threads_max = 1;
 	} else {
 		threads_max = n;
 	}
@@ -58,9 +101,21 @@ hardware_threads_get(void)
 }
 
 
+extern bool
+hardware_threads_is_mt(void)
+{
+#ifdef MYTHREAD_ENABLED
+	return threads_max > 1 || use_mt_mode_with_one_thread;
+#else
+	return false;
+#endif
+}
+
+
 extern void
 hardware_memlimit_set(uint64_t new_memlimit,
-		bool set_compress, bool set_decompress, bool is_percentage)
+		bool set_compress, bool set_decompress, bool set_mtdec,
+		bool is_percentage)
 {
 	if (is_percentage) {
 		assert(new_memlimit > 0);
@@ -110,6 +165,9 @@ hardware_memlimit_set(uint64_t new_memlimit,
 	if (set_decompress)
 		memlimit_decompress = new_memlimit;
 
+	if (set_mtdec)
+		memlimit_mtdec = new_memlimit;
+
 	return;
 }
 
@@ -117,32 +175,69 @@ hardware_memlimit_set(uint64_t new_memlimit,
 extern uint64_t
 hardware_memlimit_get(enum operation_mode mode)
 {
-	// Zero is a special value that indicates the default. Currently
-	// the default simply disables the limit. Once there is threading
-	// support, this might be a little more complex, because there will
-	// probably be a special case where a user asks for "optimal" number
-	// of threads instead of a specific number (this might even become
-	// the default mode). Each thread may use a significant amount of
-	// memory. When there are no memory usage limits set, we need some
-	// default soft limit for calculating the "optimal" number of
-	// threads.
+	// 0 is a special value that indicates the default.
+	// It disables the limit in single-threaded mode.
+	//
+	// NOTE: For multithreaded decompression, this is the hard limit
+	// (memlimit_stop). hardware_memlimit_mtdec_get() gives the
+	// soft limit (memlimit_threaded).
 	const uint64_t memlimit = mode == MODE_COMPRESS
 			? memlimit_compress : memlimit_decompress;
 	return memlimit != 0 ? memlimit : UINT64_MAX;
 }
 
 
+extern uint64_t
+hardware_memlimit_mtenc_get(void)
+{
+	return hardware_memlimit_mtenc_is_default()
+			? memlimit_mt_default
+			: hardware_memlimit_get(MODE_COMPRESS);
+}
+
+
+extern bool
+hardware_memlimit_mtenc_is_default(void)
+{
+	return memlimit_compress == 0 && threads_are_automatic;
+}
+
+
+extern uint64_t
+hardware_memlimit_mtdec_get(void)
+{
+	uint64_t m = memlimit_mtdec != 0
+			? memlimit_mtdec
+			: memlimit_mt_default;
+
+	// Cap the value to memlimit_decompress if it has been specified.
+	// This is nice for --info-memory. It wouldn't be needed for liblzma
+	// since it does this anyway.
+	if (memlimit_decompress != 0 && m > memlimit_decompress)
+		m = memlimit_decompress;
+
+	return m;
+}
+
+
 /// Helper for hardware_memlimit_show() to print one human-readable info line.
 static void
-memlimit_show(const char *str, uint64_t value)
+memlimit_show(const char *str, size_t str_columns, uint64_t value)
 {
+	// Calculate the field width so that str will be padded to take
+	// str_columns on the terminal.
+	//
+	// NOTE: If the string is invalid, this will be -1. Using -1 as
+	// the field width is fine here so it's not handled specially.
+	const int fw = tuklib_mbstr_fw(str, (int)(str_columns));
+
 	// The memory usage limit is considered to be disabled if value
 	// is 0 or UINT64_MAX. This might get a bit more complex once there
 	// is threading support. See the comment in hardware_memlimit_get().
 	if (value == 0 || value == UINT64_MAX)
-		printf("%s %s\n", str, _("Disabled"));
+		printf("  %-*s  %s\n", fw, str, _("Disabled"));
 	else
-		printf("%s %s MiB (%s B)\n", str,
+		printf("  %-*s  %s MiB (%s B)\n", fw, str,
 				uint64_to_str(round_up_to_mib(value), 0),
 				uint64_to_str(value, 1));
 
@@ -153,18 +248,60 @@ memlimit_show(const char *str, uint64_t value)
 extern void
 hardware_memlimit_show(void)
 {
+	uint32_t cputhreads = 1;
+#ifdef MYTHREAD_ENABLED
+	cputhreads = lzma_cputhreads();
+	if (cputhreads == 0)
+		cputhreads = 1;
+#endif
+
 	if (opt_robot) {
-		printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", total_ram,
-				memlimit_compress, memlimit_decompress);
+		printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64
+				"\t%" PRIu64 "\t%" PRIu32 "\n",
+				total_ram,
+				memlimit_compress,
+				memlimit_decompress,
+				hardware_memlimit_mtdec_get(),
+				memlimit_mt_default,
+				cputhreads);
 	} else {
-		// TRANSLATORS: Test with "xz --info-memory" to see if
-		// the alignment looks nice.
-		memlimit_show(_("Total amount of physical memory (RAM): "),
-				total_ram);
-		memlimit_show(_("Memory usage limit for compression:    "),
-				memlimit_compress);
-		memlimit_show(_("Memory usage limit for decompression:  "),
-				memlimit_decompress);
+		const char *msgs[] = {
+			_("Amount of physical memory (RAM):"),
+			_("Number of processor threads:"),
+			_("Compression:"),
+			_("Decompression:"),
+			_("Multi-threaded decompression:"),
+			_("Default for -T0:"),
+		};
+
+		size_t width_max = 1;
+		for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) {
+			size_t w = tuklib_mbstr_width(msgs[i], NULL);
+
+			// When debugging, catch invalid strings with
+			// an assertion. Otherwise fallback to 1 so
+			// that the columns just won't be aligned.
+			assert(w != (size_t)-1);
+			if (w == (size_t)-1)
+				w = 1;
+
+			if (width_max < w)
+				width_max = w;
+		}
+
+		puts(_("Hardware information:"));
+		memlimit_show(msgs[0], width_max, total_ram);
+		printf("  %-*s  %" PRIu32 "\n",
+				tuklib_mbstr_fw(msgs[1], (int)(width_max)),
+				msgs[1], cputhreads);
+
+		putchar('\n');
+		puts(_("Memory usage limits:"));
+		memlimit_show(msgs[2], width_max, memlimit_compress);
+		memlimit_show(msgs[3], width_max, memlimit_decompress);
+		memlimit_show(msgs[4], width_max,
+				hardware_memlimit_mtdec_get());
+		memlimit_show(msgs[5], width_max, memlimit_mt_default);
 	}
 
 	tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT);
@@ -180,7 +317,22 @@ hardware_init(void)
 	if (total_ram == 0)
 		total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
 
-	// Set the defaults.
-	hardware_memlimit_set(0, true, true, false);
+	// FIXME? There may be better methods to determine the default value.
+	// One Linux-specific suggestion is to use MemAvailable from
+	// /proc/meminfo as the starting point.
+	memlimit_mt_default = total_ram / 4;
+
+#if SIZE_MAX == UINT32_MAX
+	// A too high value may cause 32-bit xz to run out of address space.
+	// Use a conservative maximum value here. A few typical address space
+	// sizes with Linux:
+	//   - x86-64 with 32-bit xz: 4 GiB
+	//   - x86: 3 GiB
+	//   - MIPS32: 2 GiB
+	const size_t mem_ceiling = 1400U << 20;
+	if (memlimit_mt_default > mem_ceiling)
+		memlimit_mt_default = mem_ceiling;
+#endif
+
 	return;
 }