Add optional compression progress callback (#451)

This PR adds support for an optional progress reporting callback function, which is used by compression to report incremental progress as an image is processed.

The callback function is called by one of the compression threads, so doing any significant work in the callback function will slow down compression performance. E.g. emitting a progress message to stdout on a test machine slows down -fast compression by ~3%.

The command line compressor will emit a progress bar when called from a terminal, unless invoked with -silent.

Author: solidpixel

Docs/ChangeLog-4x.md

@@ -32,6 +32,11 @@ The 4.7.0 release is a maintenance release.
     is stored to an 8-bit per component file format. This option must be set
     maually for compression (`-c*`) tool operation, as the desired decode mode
     cannot be reliably determined.
+  * **Feature:** Library configuration supports a new optional progress
+    reporting callback to be specified. This is called during compression to
+    to allow interactive tooling use cases to display incremental progress. The
+    command line tool uses this feature to show compression progress unless
+    `-silent` is used.
 
 <!-- ---------------------------------------------------------------------- -->
 ## 4.6.1

Source/astcenc.h

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // ----------------------------------------------------------------------------
-// Copyright 2020-2023 Arm Limited
+// Copyright 2020-2024 Arm Limited
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy
@@ -304,6 +304,11 @@ enum astcenc_type
 	ASTCENC_TYPE_F32 = 2
 };
 
+/**
+ * @brief Function pointer type for compression progress reporting callback.
+ */
+extern "C" typedef void (*astcenc_progress_callback)(float);
+
 /**
  * @brief Enable normal map compression.
  *
@@ -566,6 +571,16 @@ struct astcenc_config
 	 */
 	float tune_search_mode0_enable;
 
+	/**
+	 * @brief The progress callback, can be @c nullptr.
+	 *
+	 * If this is specified the codec will peridocially report progress for
+	 * compression as a percentage between 0 and 100. The callback is called from one
+	 * of the compressor threads, so doing significant work in the callback will
+	 * reduce compression performance.
+	 */
+	astcenc_progress_callback progress_callback;
+
 #if defined(ASTCENC_DIAGNOSTICS)
 	/**
 	 * @brief The path to save the diagnostic trace data to.

Source/astcenc_entry.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // ----------------------------------------------------------------------------
-// Copyright 2011-2023 Arm Limited
+// Copyright 2011-2024 Arm Limited
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy
@@ -830,7 +830,7 @@ static void compress_image(
 	auto& temp_buffers = ctx.working_buffers[thread_index];
 
 	// Only the first thread actually runs the initializer
-	ctxo.manage_compress.init(block_count);
+	ctxo.manage_compress.init(block_count, ctx.config.progress_callback);
 
 	// Determine if we can use an optimized load function
 	bool needs_swz = (swizzle.r != ASTCENC_SWZ_R) || (swizzle.g != ASTCENC_SWZ_G) ||
@@ -1155,6 +1155,7 @@ astcenc_error astcenc_decompress_image(
 	unsigned int xblocks = (image_out.dim_x + block_x - 1) / block_x;
 	unsigned int yblocks = (image_out.dim_y + block_y - 1) / block_y;
 	unsigned int zblocks = (image_out.dim_z + block_z - 1) / block_z;
+	unsigned int block_count = zblocks * yblocks * xblocks;
 
 	int row_blocks = xblocks;
 	int plane_blocks = xblocks * yblocks;
@@ -1179,7 +1180,7 @@ astcenc_error astcenc_decompress_image(
 	}
 
 	// Only the first thread actually runs the initializer
-	ctxo->manage_decompress.init(zblocks * yblocks * xblocks);
+	ctxo->manage_decompress.init(block_count, nullptr);
 
 	// All threads run this processing loop until there is no work remaining
 	while (true)

Source/astcenc_internal_entry.h

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // ----------------------------------------------------------------------------
-// Copyright 2011-2022 Arm Limited
+// Copyright 2011-2024 Arm Limited
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy
@@ -118,6 +118,18 @@ class ParallelManager
 	/** @brief Number of tasks that need to be processed. */
 	unsigned int m_task_count;
 
+	/** @brief Progress callback (optional). */
+	astcenc_progress_callback m_callback;
+
+	/** @brief Lock used for callback synchronization. */
+	std::mutex m_callback_lock;
+
+	/** @brief Minimum progress before making a callback. */
+	float m_callback_min_diff;
+
+	/** @brief Last progress callback value. */
+	float m_callback_last_value;
+
 public:
 	/** @brief Create a new ParallelManager. */
 	ParallelManager()
@@ -138,6 +150,8 @@ class ParallelManager
 		m_start_count = 0;
 		m_done_count = 0;
 		m_task_count = 0;
+		m_callback_last_value = 0.0f;
+		m_callback_min_diff = 1.0f;
 	}
 
 	/**
@@ -166,14 +180,20 @@ class ParallelManager
 	 * initialization. Other threads will block and wait for it to complete.
 	 *
 	 * @param task_count   Total number of tasks needing processing.
+	 * @param callback     Function pointer for progress status callbacks.
 	 */
-	void init(unsigned int task_count)
+	void init(unsigned int task_count, astcenc_progress_callback callback)
 	{
 		std::lock_guard<std::mutex> lck(m_lock);
 		if (!m_init_done)
 		{
+			m_callback = callback;
 			m_task_count = task_count;
 			m_init_done = true;
+
+			// Report every 1% or 4096 blocks, whichever is larger, to avoid callback overhead
+			float min_diff = (4096.0f / static_cast<float>(task_count)) * 100.0f;
+			m_callback_min_diff = astc::max(min_diff, 1.0f);
 		}
 	}
 
@@ -212,12 +232,49 @@ class ParallelManager
 	{
 		// Note: m_done_count cannot use an atomic without the mutex; this has a race between the
 		// update here and the wait() for other threads
-		std::unique_lock<std::mutex> lck(m_lock);
-		this->m_done_count += count;
-		if (m_done_count == m_task_count)
+		unsigned int local_count;
+		float local_last_value;
 		{
-			lck.unlock();
-			m_complete.notify_all();
+			std::unique_lock<std::mutex> lck(m_lock);
+			m_done_count += count;
+			local_count = m_done_count;
+			local_last_value = m_callback_last_value;
+
+			if (m_done_count == m_task_count)
+			{
+				// Ensure the progress bar hits 100%
+				if (m_callback)
+				{
+					std::unique_lock<std::mutex> cblck(m_callback_lock);
+					m_callback(100.0f);
+					m_callback_last_value = 100.0f;
+				}
+
+				lck.unlock();
+				m_complete.notify_all();
+			}
+		}
+
+		// Process progress callback if we have one
+		if (m_callback)
+		{
+			// Initial lockless test - have we progressed enough to emit?
+			float num = static_cast<float>(local_count);
+			float den = static_cast<float>(m_task_count);
+			float this_value =  (num / den) * 100.0f;
+			bool report_test = (this_value - local_last_value) > m_callback_min_diff;
+
+			// Recheck under lock, because another thread might report first
+			if (report_test)
+			{
+				std::unique_lock<std::mutex> cblck(m_callback_lock);
+				bool report_retest = (this_value - m_callback_last_value) > m_callback_min_diff;
+				if (report_retest)
+				{
+					m_callback(this_value);
+					m_callback_last_value = this_value;
+				}
+			}
 		}
 	}
 

Source/astcenccli_toplevel.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // ----------------------------------------------------------------------------
-// Copyright 2011-2023 Arm Limited
+// Copyright 2011-2024 Arm Limited
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy
@@ -22,6 +22,12 @@
 #include "astcenc.h"
 #include "astcenccli_internal.h"
 
+#if defined(_WIN32)
+	#include <io.h>
+	#define isatty _isatty
+#else
+	#include <unistd.h>
+#endif
 #include <cassert>
 #include <cstring>
 #include <functional>
@@ -156,6 +162,35 @@ struct decompression_workload
 	astcenc_error error;
 };
 
+/**
+ * @brief Callback emitting a progress bar
+ */
+extern "C" void progress_emitter(
+	float value
+) {
+	const unsigned int bar_size = 25;
+	unsigned int parts = static_cast<int>(value / 4.0f);
+
+	char buffer[bar_size + 3];
+	buffer[0] = '[';
+
+	for (unsigned int i = 0; i < parts; i++)
+	{
+		buffer[i + 1] = '=';
+	}
+
+	for (unsigned int i = parts; i < bar_size; i++)
+	{
+		buffer[i + 1] = ' ';
+	}
+
+	buffer[bar_size + 1] = ']';
+	buffer[bar_size + 2] = '\0';
+
+	printf("    Progress: %s %03.1f%%\r", buffer, static_cast<double>(value));
+	fflush(stdout);
+}
+
 /**
  * @brief Test if a string argument is a well formed float.
  */
@@ -1954,6 +1989,18 @@ int astcenc_main(
 		return 1;
 	}
 
+	// Enable progress callback if not in silent mode and using a terminal
+	#if defined(_WIN32)
+		int stdoutfno = _fileno(stdout);
+	#else
+		int stdoutfno = STDOUT_FILENO;
+	#endif
+
+	if ((!cli_config.silentmode) && isatty(stdoutfno))
+	{
+		config.progress_callback = progress_emitter;
+	}
+
 	astcenc_image* image_uncomp_in = nullptr ;
 	unsigned int image_uncomp_in_component_count = 0;
 	bool image_uncomp_in_is_hdr = false;
@@ -2122,6 +2169,13 @@ int astcenc_main(
 		double start_compression_time = get_time();
 		for (unsigned int i = 0; i < cli_config.repeat_count; i++)
 		{
+			if (config.progress_callback)
+			{
+				printf("Compression\n");
+				printf("===========\n");
+				printf("\n");
+			}
+
 			double start_iter_time = get_time();
 			if (cli_config.thread_count > 1)
 			{
@@ -2136,6 +2190,11 @@ int astcenc_main(
 
 			astcenc_compress_reset(codec_context);
 
+			if (config.progress_callback)
+			{
+				printf("\n\n");
+			}
+
 			double iter_time = get_time() - start_iter_time;
 			best_compression_time = astc::min(iter_time, best_compression_time);
 		}

jenkins/nightly.Jenkinsfile

@@ -204,6 +204,8 @@ pipeline {
                   cd build_rel_arm64
                   cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64-clangcl ..
                   msbuild astcencoder.sln -property:Configuration=Release
+                  msbuild PACKAGE.vcxproj -property:Configuration=Release
+                  msbuild INSTALL.vcxproj -property:Configuration=Release
                 '''
               }
             }