diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dd0ddb0a..1ad94a68 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,5 @@ +# [qtAliceVision] Reformat all sequence cache-related files +d304d8655de26ed5702966b4d855b031f85268f1 # [qtAliceVision] Apply clang-format on modified files d0fd33d78af7d73f4c1c5ba0de7a1d49d263f5d0 # Reformat all plugins with latest clang-format rules diff --git a/src/qtAliceVision/AsyncFetcher.cpp b/src/qtAliceVision/AsyncFetcher.cpp new file mode 100644 index 00000000..ce4b649d --- /dev/null +++ b/src/qtAliceVision/AsyncFetcher.cpp @@ -0,0 +1,247 @@ +#include "AsyncFetcher.hpp" + +#include +#include + +#include +#include + +using namespace aliceVision; + +namespace qtAliceVision { +namespace imgserve { + +AsyncFetcher::AsyncFetcher() +{ + _resizeRatio = 0.001; + _isAsynchronous = false; + _requestSynchronous = false; +} + +AsyncFetcher::~AsyncFetcher() {} + +void AsyncFetcher::setSequence(const std::vector& paths) +{ + // Sequence can't be changed while thread is running + if (_isAsynchronous) + { + return; + } + + _sequence = paths; + _currentIndex = 0; + + for (unsigned idx = 0; idx < _sequence.size(); idx++) + { + _pathToSeqId[_sequence[idx]] = idx; + } +} + +void AsyncFetcher::setResizeRatio(double ratio) +{ + QMutexLocker locker(&_mutexResizeRatio); + _resizeRatio = ratio; +} + +void AsyncFetcher::setCache(ImageCache::uptr&& cache) +{ + // Cache can't be changed while thread is running + if (_isAsynchronous) + { + return; + } + _cache = std::move(cache); +} + +void AsyncFetcher::run() +{ + using namespace std::chrono_literals; + + _isAsynchronous = true; + _requestSynchronous = false; + + std::size_t previousCacheSize = getDiskLoads(); + + while (1) + { + if (_requestSynchronous) + { + _requestSynchronous = false; + break; + } + + if (_sequence.size() == 0) + { + std::this_thread::sleep_for(100ms); + } + else + { + const std::string& lpath = _sequence[static_cast(_currentIndex)]; + + // Load in cache + if (_cache) + { + double ratio; + { + QMutexLocker locker(&_mutexResizeRatio); + ratio = _resizeRatio; + } + + _cache->get(lpath, static_cast(_currentIndex), ratio, false); + } + + _currentIndex++; + + int size = static_cast(_sequence.size()); + if (_currentIndex >= size) + { + _currentIndex = 0; + } + + std::this_thread::sleep_for(1ms); + } + + std::size_t cacheSize = getDiskLoads(); + if (cacheSize != previousCacheSize) + { + previousCacheSize = cacheSize; + Q_EMIT onAsyncFetchProgressed(); + } + } + + _requestSynchronous = false; + _isAsynchronous = false; +} + +void AsyncFetcher::stopAsync() +{ + _requestSynchronous = true; +} + +void AsyncFetcher::updateCacheMemory(std::size_t maxMemory) +{ + if (_cache) + { + _cache->updateMaxMemory(maxMemory); + } +} + +std::size_t AsyncFetcher::getCacheSize() const +{ + return (_cache) ? static_cast(_cache->info().getContentSize()) : 0; +} + +std::size_t AsyncFetcher::getDiskLoads() const +{ + return (_cache) ? static_cast(_cache->info().getLoadFromDisk()) : 0; +} + +QVariantList AsyncFetcher::getCachedFrames() const +{ + QVariantList intervals; + + if (!_cache) + { + return intervals; + } + + // Accumulator variables + auto region = std::make_pair(-1, -1); + bool regionOpen = false; + + size_t size = _sequence.size(); + + { + // Build cached frames intervals + for (std::size_t i = 0; i < size; ++i) + { + const int frame = static_cast(i); + + // Check if current frame is in cache + if (_cache->contains(_sequence[i], _resizeRatio)) + { + // Either grow currently open region or create a new region + if (regionOpen) + { + region.second = frame; + } + else + { + region.first = frame; + region.second = frame; + regionOpen = true; + } + } + else + { + // Close currently open region + if (regionOpen) + { + intervals.append(QPoint(region.first, region.second)); + regionOpen = false; + } + } + } + } + + // Last region may still be open + if (regionOpen) + { + intervals.append(QPoint(region.first, region.second)); + } + + return intervals; +} + +bool AsyncFetcher::getFrame(const std::string& path, + std::shared_ptr>& image, + oiio::ParamValueList& metadatas, + size_t& originalWidth, + size_t& originalHeight) +{ + // Need a cache + if (!_cache) + { + return false; + } + + // First try getting the image + bool onlyCache = _isAsynchronous; + + // Upgrade the thread with the current Index + for (std::size_t idx = 0; idx < _sequence.size(); ++idx) + { + if (_sequence[idx] == path) + { + _currentIndex = static_cast(idx); + break; + } + } + + std::optional ovalue = _cache->get(path, _currentIndex, _resizeRatio, onlyCache); + + if (ovalue.has_value()) + { + auto& value = ovalue.value(); + image = value.get(); + + oiio::ParamValueList copy_metadatas = value.getMetadatas(); + metadatas = copy_metadatas; + originalWidth = value.getOriginalWidth(); + originalHeight = value.getOriginalHeight(); + + if (image) + { + _cache->setReferenceFrameId(_currentIndex); + } + + return true; + } + + return false; +} + +} // namespace imgserve +} // namespace qtAliceVision + +#include "AsyncFetcher.moc" diff --git a/src/qtAliceVision/AsyncFetcher.hpp b/src/qtAliceVision/AsyncFetcher.hpp new file mode 100644 index 00000000..80ee1a1d --- /dev/null +++ b/src/qtAliceVision/AsyncFetcher.hpp @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include +#include + +#include "ImageCache.hpp" + +namespace qtAliceVision { +namespace imgserve { + +class AsyncFetcher : public QObject, public QRunnable +{ + Q_OBJECT + + public: + AsyncFetcher(); + ~AsyncFetcher(); + + /** + * @brief Cache object is created externally. + * Pass it to the Fetcher for use (Fetcher get ownership) + * @param cache the cache object to store + */ + void setCache(ImageCache::uptr&& cache); + + /** + * @brief set the image sequence + * The image sequence is a list of image paths which is ordered + * The Fetcher must not be in asynchronous mode for this function to work + * As such, the _sequence object is only used in read mode during async mode. + */ + void setSequence(const std::vector& paths); + + /** + * @brief update the resize ratio of the image + * @param ratio the coefficient of resize of the loaded images + */ + void setResizeRatio(double ratio); + + /** + * @brief retrieve a frame from the cache in both sync and async mode + * @param path the image path which should be contained in _sequence. + * @param image the result image pointer + * @param metadatas the image metadatas found in the file + * @param originalWidth the image width before the resize + * @param originalHeight the image height before the resize + * @return true if the image was succesfully found in the cache + */ + bool getFrame(const std::string& path, + std::shared_ptr>& image, + oiio::ParamValueList& metadatas, + std::size_t& originalWidth, + std::size_t& originalHeight); + + /** + * @brief Internal function for QT to start the asynchronous mode + */ + Q_SLOT void run() override; + + /** + * @brief stop asynchronous mode + * The caller have to wait on the thread pool to guarantee the effective end + */ + void stopAsync(); + + bool isAsync() const { return _isAsynchronous; } + + /** + * @brief get the cache content size in bytes + * @return the cache content size in bytes + */ + std::size_t getCacheSize() const; + + /** + * @brief get the number of images loaded + * @return the count of images loaded since the creation of the cache object + */ + std::size_t getDiskLoads() const; + + /** + * @brief update maxMemory for the cache + * @param maxMemory the number of bytes allowed in the cache + */ + void updateCacheMemory(std::size_t maxMemory); + + /** + * @brief get a list of regions containing the image frames + * @return a list of two values (begin, end) + */ + QVariantList getCachedFrames() const; + + public: + Q_SIGNAL void onAsyncFetchProgressed(); + + private: + ImageCache::uptr _cache; + + std::vector _sequence; + std::unordered_map _pathToSeqId; + + QAtomicInt _currentIndex; + QAtomicInt _isAsynchronous; + QAtomicInt _requestSynchronous; + + double _resizeRatio; + QMutex _mutexResizeRatio; +}; + +} // namespace imgserve +} // namespace qtAliceVision diff --git a/src/qtAliceVision/CMakeLists.txt b/src/qtAliceVision/CMakeLists.txt index 4b672118..6d22aea5 100644 --- a/src/qtAliceVision/CMakeLists.txt +++ b/src/qtAliceVision/CMakeLists.txt @@ -13,6 +13,8 @@ set(PLUGIN_SOURCES Painter.cpp SequenceCache.cpp SingleImageLoader.cpp + ImageCache.cpp + AsyncFetcher.cpp ) set(PLUGIN_HEADERS @@ -32,6 +34,9 @@ set(PLUGIN_HEADERS ImageServer.hpp SequenceCache.hpp SingleImageLoader.hpp + ImageCache.hpp + AsyncFetcher.hpp + ) set(PLUGIN_MOCS @@ -43,6 +48,7 @@ set(PLUGIN_MOCS MSfMDataStats.hpp SequenceCache.hpp SingleImageLoader.hpp + AsyncFetcher.hpp ) diff --git a/src/qtAliceVision/FloatImageViewer.cpp b/src/qtAliceVision/FloatImageViewer.cpp index 63b6b97f..bb373d11 100644 --- a/src/qtAliceVision/FloatImageViewer.cpp +++ b/src/qtAliceVision/FloatImageViewer.cpp @@ -44,7 +44,6 @@ FloatImageViewer::FloatImageViewer(QQuickItem* parent) connect(&_singleImageLoader, &imgserve::SingleImageLoader::requestHandled, this, &FloatImageViewer::reload); connect(&_sequenceCache, &imgserve::SequenceCache::requestHandled, this, &FloatImageViewer::reload); - connect(&_sequenceCache, &imgserve::SequenceCache::contentChanged, this, &FloatImageViewer::reload); connect(this, &FloatImageViewer::useSequenceChanged, this, &FloatImageViewer::reload); } @@ -78,14 +77,24 @@ void FloatImageViewer::setSequence(const QVariantList& paths) void FloatImageViewer::setFetchingSequence(bool fetching) { - _sequenceCache.setFetchingSequence(fetching); + _sequenceCache.setAsyncFetching(fetching); Q_EMIT fetchingSequenceChanged(); } void FloatImageViewer::setTargetSize(int size) { - _sequenceCache.setTargetSize(size); - Q_EMIT targetSizeChanged(); +} + +void FloatImageViewer::setResizeRatio(double ratio) +{ + ratio = std::clamp(ratio, 0.0, 1.0); + ratio = std::ceil(ratio * 10.0) / 10.0; + + _sequenceCache.setResizeRatio(ratio); + + _clampedResizeRatio = ratio; + + Q_EMIT resizeRatioChanged(); } void FloatImageViewer::setMemoryLimit(int memoryLimit) { @@ -166,18 +175,21 @@ void FloatImageViewer::reload() qWarning() << "[QtAliceVision] The loading status has not been updated since the last reload. Something wrong might have happened."; setStatus(EStatus::OUTDATED_LOADING); } - Q_EMIT cachedFramesChanged(); } void FloatImageViewer::playback(bool active) { - // Turn off interactive prefetching when playback is ON - _sequenceCache.setInteractivePrefetching(!active); } QVector4D FloatImageViewer::pixelValueAt(int x, int y) { + if (_useSequence) + { + x = int(std::ceil(double(x) * _clampedResizeRatio)); + y = int(std::ceil(double(y) * _clampedResizeRatio)); + } + if (!_image) { // qInfo() << "[QtAliceVision] FloatImageViewer::pixelValueAt(" << x << ", " << y << ") => no valid image"; diff --git a/src/qtAliceVision/FloatImageViewer.hpp b/src/qtAliceVision/FloatImageViewer.hpp index 5aa110f6..9194d915 100644 --- a/src/qtAliceVision/FloatImageViewer.hpp +++ b/src/qtAliceVision/FloatImageViewer.hpp @@ -59,6 +59,8 @@ class FloatImageViewer : public QQuickItem Q_PROPERTY(int targetSize WRITE setTargetSize NOTIFY targetSizeChanged) + Q_PROPERTY(double resizeRatio WRITE setResizeRatio NOTIFY resizeRatioChanged) + Q_PROPERTY(QVariantList cachedFrames READ getCachedFrames NOTIFY cachedFramesChanged) Q_PROPERTY(bool useSequence MEMBER _useSequence NOTIFY useSequenceChanged) @@ -139,6 +141,7 @@ class FloatImageViewer : public QQuickItem Q_SIGNAL void fisheyeCircleParametersChanged(); Q_SIGNAL void sequenceChanged(); Q_SIGNAL void targetSizeChanged(); + Q_SIGNAL void resizeRatioChanged(); Q_SIGNAL void cachedFramesChanged(); Q_SIGNAL void useSequenceChanged(); Q_SIGNAL void fetchingSequenceChanged(); @@ -154,6 +157,8 @@ class FloatImageViewer : public QQuickItem void setTargetSize(int size); + void setResizeRatio(double ratio); + void setFetchingSequence(bool fetching); void setMemoryLimit(int memoryLimit); @@ -202,6 +207,7 @@ class FloatImageViewer : public QQuickItem imgserve::SequenceCache _sequenceCache; imgserve::SingleImageLoader _singleImageLoader; bool _useSequence = true; + double _clampedResizeRatio = 1.0; }; } // namespace qtAliceVision diff --git a/src/qtAliceVision/ImageCache.cpp b/src/qtAliceVision/ImageCache.cpp new file mode 100644 index 00000000..420a64cb --- /dev/null +++ b/src/qtAliceVision/ImageCache.cpp @@ -0,0 +1,96 @@ +#include "ImageCache.hpp" + +namespace qtAliceVision { + +ImageCache::ImageCache(unsigned long maxSize, const aliceVision::image::ImageReadOptions& options) + : _info(maxSize), + _options(options), + _referenceFrameId(0) +{} + +ImageCache::~ImageCache() {} + +void ImageCache::cleanup(size_t requestedSize, const CacheKey& toAdd) +{ + // At each step, we try to remove the LRU item which is not used + while (1) + { + // Check if we did enough work? + size_t available = _info.getAvailableSize(); + if (available >= requestedSize) + { + return; + } + + bool erased = false; + + // First, try to remove images with different ratios + { + std::scoped_lock lockKeys(_mutexAccessImages); + for (const auto& [key, value] : _imagePtrs) + { + if (key.resizeRatio == toAdd.resizeRatio) + { + continue; + } + + if (value.useCount() <= 1) + { + _imagePtrs.erase(key); + _info.update(_imagePtrs); + erased = true; + break; + } + } + } + + // If we get here, all the cache should contain only the same resize ratio + if (!erased) + { + std::scoped_lock lockKeys(_mutexAccessImages); + + std::map orderedKeys; + + for (const auto& [key, value] : _imagePtrs) + { + int iOtherId = int(value.getFrameId()); + int diff = iOtherId - _referenceFrameId; + + // Before the frameId, difference is negative. + // The closest it is to the frameId before the frameId, the highest its priority to delete + // After the frameId, the largest the difference, the highest its priority to delete + if (diff < 0) + { + diff = std::numeric_limits::max() + diff; + } + + orderedKeys[diff] = &key; + } + + if (orderedKeys.size() > 0) + { + const CacheKey* pKey = orderedKeys.rbegin()->second; + _imagePtrs.erase(*pKey); + _info.update(_imagePtrs); + } + } + + // Nothing happened, nothing more will happen. + if (!erased) + { + return; + } + } +} + +void ImageCache::updateMaxMemory(unsigned long long int maxSize) +{ + _info.setMaxMemory(maxSize); +} + +void ImageCache::setReferenceFrameId(int referenceFrameId) +{ + _referenceFrameId = referenceFrameId; +} + +} // namespace qtAliceVision diff --git a/src/qtAliceVision/ImageCache.hpp b/src/qtAliceVision/ImageCache.hpp new file mode 100644 index 00000000..4951f557 --- /dev/null +++ b/src/qtAliceVision/ImageCache.hpp @@ -0,0 +1,434 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include + + +namespace qtAliceVision { +/** + * @brief A struct used to identify a cached image using its file description, color type info and downscale level. + */ +struct CacheKey +{ + std::string filename; + int nbChannels; + oiio::TypeDesc::BASETYPE typeDesc; + double resizeRatio; + std::time_t lastWriteTime; + + CacheKey(const std::string& path, int nchannels, oiio::TypeDesc::BASETYPE baseType, double ratio, std::time_t time) + : filename(path), + nbChannels(nchannels), + typeDesc(baseType), + resizeRatio(ratio), + lastWriteTime(time) + {} + + bool operator==(const CacheKey& other) const + { + return (filename == other.filename && nbChannels == other.nbChannels && typeDesc == other.typeDesc && resizeRatio == other.resizeRatio && + lastWriteTime == other.lastWriteTime); + } +}; + +struct CacheKeyHasher +{ + std::size_t operator()(const CacheKey& key) const noexcept + { + std::size_t seed = 0; + boost::hash_combine(seed, key.filename); + boost::hash_combine(seed, key.nbChannels); + boost::hash_combine(seed, key.typeDesc); + boost::hash_combine(seed, key.resizeRatio); + boost::hash_combine(seed, key.lastWriteTime); + return seed; + } +}; + +/** + * @brief A class to support shared pointers for all types of images. + */ +class CacheValue +{ + public: + template + CacheValue(unsigned frameId, std::shared_ptr> img) + : _vimg(img), + _frameId(frameId) + {} + + public: + /** + * @brief Template method to get a shared pointer to the image with pixel type given as template argument. + * @note At most one of the generated methods will provide a non-null pointer. + * @return shared pointer to an image with the pixel type given as template argument + */ + template + std::shared_ptr> get() const + { + return std::get>>(_vimg); + } + + unsigned getOriginalWidth() const + { + return _originalWidth; + } + + unsigned getOriginalHeight() const + { + return _originalHeight; + } + + void setOriginalWidth(unsigned width) + { + _originalWidth = width; + } + + void setOriginalHeight(unsigned height) + { + _originalHeight = height; + } + + oiio::ParamValueList & getMetadatas() + { + return _metadatas; + } + + unsigned getFrameId() const + { + return _frameId; + } + + /** + * @brief Count the number of usages of the wrapped shared pointer. + * @return the use_count of the wrapped shared pointer if there is one, otherwise 0 + */ + long int useCount() const + { + return std::visit([](const auto& arg) -> long int { return arg.use_count(); }, _vimg); + } + + /** + * @brief Retrieve the memory size (in bytes) of the wrapped image. + * @return the memory size of the wrapped image if there is one, otherwise 0 + */ + unsigned long long int memorySize() const + { + return std::visit([](const auto& arg) -> unsigned long long int { return arg->memorySize(); }, _vimg); + } + + private: + std::variant>, + std::shared_ptr>, + std::shared_ptr>, + std::shared_ptr>, + std::shared_ptr>, + std::shared_ptr>> + _vimg; + + unsigned _originalWidth; + unsigned _originalHeight; + oiio::ParamValueList _metadatas; + unsigned _frameId; +}; + +/** + * @brief A struct to store information about the cache current state and usage. + */ +class CacheInfo +{ + public: + CacheInfo(unsigned long int maxSize) + : _maxSize(maxSize) + {} + + void incrementCache() + { + const std::scoped_lock lockPeek(_mutex); + _nbLoadFromCache++; + } + + void incrementDisk() + { + const std::scoped_lock lock(_mutex); + _nbLoadFromDisk++; + } + + unsigned long long int getCapacity() const + { + const std::scoped_lock lock(_mutex); + return _maxSize; + } + + void update(const std::unordered_map& images) + { + std::scoped_lock lock(_mutex); + + _contentSize = 0; + for (const auto& [key, value] : images) + { + _contentSize += value.memorySize(); + _nbImages++; + } + } + + std::size_t getAvailableSize() const + { + const std::scoped_lock lock(_mutex); + + if (_maxSize <= _contentSize) + { + return 0; + } + + return _maxSize - _contentSize; + } + + bool isSmallEnough(std::size_t value) const + { + const std::scoped_lock lock(_mutex); + + return (_contentSize + value < _maxSize); + } + + unsigned long long int getContentSize() const + { + const std::scoped_lock lock(_mutex); + return _contentSize; + } + + int getLoadFromDisk() const + { + const std::scoped_lock lock(_mutex); + return _nbLoadFromDisk; + } + + void setMaxMemory(unsigned long long int maxSize) + { + std::scoped_lock lock(_mutex); + _maxSize = maxSize; + } + + /// memory usage limits + unsigned long long int _maxSize; + + /// current state of the cache + int _nbImages = 0; + unsigned long long int _contentSize = 0; + + /// usage statistics + int _nbLoadFromDisk = 0; + int _nbLoadFromCache = 0; + int _nbRemoveUnused = 0; + + mutable std::mutex _mutex; +}; + +class ImageCache +{ + public: + using uptr = std::unique_ptr; + + public: + /** + * @brief Create a new image cache by defining memory usage limits and image reading options. + * @param[in] maxSize the cache maximal size (in bytes) + * @param[in] options the reading options that will be used when loading images through this cache + */ + ImageCache(unsigned long maxSize, const aliceVision::image::ImageReadOptions& options); + + /** + * @brief Destroy the cache and the unused images it contains. + */ + ~ImageCache(); + + /// make image cache class non-copyable + ImageCache(const ImageCache&) = delete; + ImageCache& operator=(const ImageCache&) = delete; + + /** + * @brief Retrieve a cached image at a given downscale level. + * @note This method is thread-safe. + * @param[in] filename the image's filename on disk + * @param[in] frameId additional data + * @param[in] resizeRatio the resize ratio of the image + * @param[in] cachedOnly if true, only return images that are already in the cache + * @return a shared pointer to the cached image + */ + template + std::optional get(const std::string& filename, unsigned frameId, double resizeRatio = 1.0, bool cachedOnly = false); + + /** + * @brief Check if an image at a given downscale level is currently in the cache. + * @note This method is thread-safe. + * @param[in] filename the image's filename on disk + * @param[in] resizeRatio the resize ratio of the image + * @return whether or not the cache currently contains the image + */ + template + bool contains(const std::string& filename, double resizeRatio = 1.0) const; + + /** + * Ask for more room, by deleting the LRU items which are not used + * @param requestedSize the required size for the new image + * @param toAdd the key of the image to add after cleanup + */ + void cleanup(std::size_t requestedSize, const CacheKey& toAdd); + + /** + * @return information on the current cache state and usage + */ + inline const CacheInfo& info() const { return _info; } + + /** + * @return the image reading options of the cache + */ + inline const aliceVision::image::ImageReadOptions& readOptions() const { return _options; } + + /** + * @brief update the cache max memory + * @param maxSize the value to store + */ + void updateMaxMemory(unsigned long long int maxSize); + + /** + * @brief set the reference frame ID + * @param referenceFrameId the value to store + */ + void setReferenceFrameId(int referenceFrameId); + + private: + /** + * @brief Load a new image corresponding to the given key and add it as a new entry in the cache. + * @param[in] key the key used to identify the entry in the cache + * @param[in] frameId additional data + */ + template + std::optional load(const CacheKey& key, unsigned frameId); + + CacheInfo _info; + aliceVision::image::ImageReadOptions _options; + + // Set of images stored and indexed by CacheKey + std::unordered_map _imagePtrs; + mutable std::mutex _mutexAccessImages; + + // Reference frame Id used to compute the next image to remove + // This should be equal to the currently displayed image + std::atomic _referenceFrameId; +}; + +// Since some methods in the ImageCache class are templated +// their definition must be given in this header file + +template +std::optional ImageCache::get(const std::string& filename, unsigned frameId, double resizeRatio, bool cachedOnly) +{ + if (resizeRatio < 1e-12) + { + return std::nullopt; + } + + // Build lookup key + using TInfo = aliceVision::image::ColorTypeInfo; + auto lastWriteTime = aliceVision::utils::getLastWriteTime(filename); + CacheKey keyReq(filename, TInfo::size, TInfo::typeDesc, resizeRatio, lastWriteTime); + + // find the requested image in the cached images + { + std::scoped_lock lockImages(_mutexAccessImages); + auto it = _imagePtrs.find(keyReq); + if (it != _imagePtrs.end()) + { + return it->second; + } + } + + if (cachedOnly) + { + return std::nullopt; + } + + // Load image and add to cache if possible + return load(keyReq, frameId); +} + +template +std::optional ImageCache::load(const CacheKey& key, unsigned frameId) +{ + aliceVision::image::Image img; + auto resized = std::make_shared>(); + + int width = 0; + int height = 0; + oiio::ParamValueList metadatas; + + try + { + metadatas = aliceVision::image::readImageMetadata(key.filename, width, height); + + // load image from disk + readImage(key.filename, img, _options); + } + catch (...) + { + return std::nullopt; + } + + // Compute new size, make sure the size is at least 1 + double dw = key.resizeRatio * double(img.width()); + double dh = key.resizeRatio * double(img.height()); + int tw = static_cast(std::max(1, int(std::ceil(dw)))); + int th = static_cast(std::max(1, int(std::ceil(dh)))); + + using TInfo = aliceVision::image::ColorTypeInfo; + cleanup(tw * th * std::size_t(TInfo::size), key); + + // apply downscale + aliceVision::imageAlgo::resizeImage(tw, th, img, *resized); + + // Increment disk access stats + _info.incrementDisk(); + + // create wrapper around shared pointer + CacheValue value(frameId, resized); + + // Add additional information about the image + value.setOriginalHeight(static_cast(height)); + value.setOriginalWidth(static_cast(width)); + value.getMetadatas() = metadatas; + + // Store image in map + { + std::scoped_lock lockImages(_mutexAccessImages); + + _imagePtrs.insert({key, value}); + _info.update(_imagePtrs); + } + + return value; +} + +template +bool ImageCache::contains(const std::string& filename, double resizeRatio) const +{ + std::scoped_lock lockKeys(_mutexAccessImages); + + using TInfo = aliceVision::image::ColorTypeInfo; + auto lastWriteTime = aliceVision::utils::getLastWriteTime(filename); + CacheKey keyReq(filename, TInfo::size, TInfo::typeDesc, resizeRatio, lastWriteTime); + auto it = _imagePtrs.find(keyReq); + + bool found = (it != _imagePtrs.end()); + + return found; +} + +} // namespace qtAliceVision diff --git a/src/qtAliceVision/SequenceCache.cpp b/src/qtAliceVision/SequenceCache.cpp index 487e8f57..e6613ba4 100644 --- a/src/qtAliceVision/SequenceCache.cpp +++ b/src/qtAliceVision/SequenceCache.cpp @@ -2,28 +2,16 @@ #include -#include -#include - -#include -#include -#include -#include -#include -#include -#include +using namespace aliceVision; namespace qtAliceVision { namespace imgserve { -// Flag for aborting the prefetching worker thread from the main thread -std::atomic_bool abortPrefetching = false; - SequenceCache::SequenceCache(QObject* parent) : QObject(parent) { // Retrieve memory information from system - const auto memInfo = aliceVision::system::getMemoryInfo(); + const auto memInfo = system::getMemoryInfo(); // Compute proportion of RAM that can be dedicated to image caching // For now we use 30% of available RAM @@ -31,195 +19,86 @@ SequenceCache::SequenceCache(QObject* parent) const double cacheRatio = 0.3; const double cacheRam = cacheRatio * availableRam; - // Initialize image cache - const double factorConvertMiB = 1024. * 1024.; - const float fCacheRam = static_cast(cacheRam / factorConvertMiB); - _cache = new aliceVision::image::ImageCache(fCacheRam, fCacheRam, aliceVision::image::EImageColorSpace::LINEAR); + _maxMemory = static_cast(cacheRam); - // Initialize internal state - _regionSafe = std::make_pair(-1, -1); - _loading = false; - _interactivePrefetching = true; - _targetSize = 1000; - _fetchingSequence = false; -} + _fetcher.setAutoDelete(false); -SequenceCache::~SequenceCache() -{ - // Check if a worker thread is currently active - if (_loading) + // Cache does not exist + // Let's create a new one! { - // Worker thread will return on next iteration - abortPrefetching = true; - _threadPool.waitForDone(); + ImageCache::uptr cache = std::make_unique(_maxMemory, image::EImageColorSpace::LINEAR); + _fetcher.setCache(std::move(cache)); } - - // Free memory occupied by image cache - if (_cache) - delete _cache; } -void SequenceCache::setSequence(const QVariantList& paths) +SequenceCache::~SequenceCache() { - _lockSequence.lock(); - { - _sequenceId++; - - abortPrefetching = true; - _threadPool.waitForDone(); - abortPrefetching = false; - - // Clear internal state - _sequence.clear(); - _regionSafe = std::make_pair(-1, -1); - - // Fill sequence vector - int frameCounter = 0; - for (const auto& var : paths) - { - try - { - // Initialize frame data - FrameData data; - data.path = var.toString().toStdString(); - - // Retrieve metadata from disk - int width, height; - auto metadata = aliceVision::image::readImageMetadata(data.path, width, height); - - // Store original image dimensions - data.dim = QSize(width, height); - - // Copy metadata into a QVariantMap - for (const auto& item : metadata) - { - data.metadata[QString::fromStdString(item.name().string())] = QString::fromStdString(item.get_string()); - } - - // Compute downscale - const int maxDim = std::max(width, height); - const int level = static_cast(std::floor(std::log2(static_cast(maxDim) / static_cast(_targetSize)))); - data.downscale = 1 << std::max(level, 0); - - // Set frame number - data.frame = frameCounter; - - // Add to sequence - _sequence.push_back(data); - ++frameCounter; - } - catch (const std::runtime_error& e) - { - // Log error - std::cerr << e.what() << std::endl; - } - } - } - _lockSequence.unlock(); - - // Notify listeners that sequence content has changed - Q_EMIT contentChanged(); + _fetcher.stopAsync(); + _threadPool.waitForDone(); } -void SequenceCache::setInteractivePrefetching(bool interactive) { _interactivePrefetching = interactive; } - -void SequenceCache::setTargetSize(int size) +void SequenceCache::setSequence(const QVariantList& paths) { - // Update target size - _targetSize = size; - - // Update downscale for each frame - bool refresh = false; - for (auto& data : _sequence) - { - // Compute downscale - const int maxDim = std::max(data.dim.width(), data.dim.height()); - const int level = static_cast(std::floor(std::log2(static_cast(maxDim) / static_cast(_targetSize)))); - const int downscale = 1 << std::max(level, 0); + bool isAsync = _fetcher.isAsync(); - refresh = refresh || (data.downscale != downscale); + _fetcher.stopAsync(); + _threadPool.waitForDone(); - data.downscale = downscale; + // Convert to string + std::vector sequence; + for (const auto& item : paths) + { + sequence.push_back(item.toString().toStdString()); } - if (refresh) - { - // Clear internal state - _regionSafe = std::make_pair(-1, -1); + // Assign sequence to fetcher + _fetcher.setSequence(sequence); - // Notify listeners that sequence content has changed - Q_EMIT contentChanged(); - } + // Restart if needed + setAsyncFetching(isAsync); } -QVariantList SequenceCache::getCachedFrames() const +void SequenceCache::setResizeRatio(double ratio) { - QVariantList intervals; - - // Accumulator variables - auto region = std::make_pair(-1, -1); - bool regionOpen = false; - - // Build cached frames intervals - for (std::size_t i = 0; i < _sequence.size(); ++i) - { - const int frame = static_cast(i); - - // Check if current frame is in cache - if (_cache->contains(_sequence[i].path, _sequence[i].downscale)) - { - // Either grow currently open region or create a new region - if (regionOpen) - { - region.second = frame; - } - else - { - region.first = frame; - region.second = frame; - regionOpen = true; - } - } - else - { - // Close currently open region - if (regionOpen) - { - intervals.append(QPoint(region.first, region.second)); - regionOpen = false; - } - } - } - - // Last region may still be open - if (regionOpen) - { - intervals.append(QPoint(region.first, region.second)); - } + _fetcher.setResizeRatio(ratio); +} - return intervals; +void SequenceCache::setMemoryLimit(int memory) +{ + // Convert parameter in gigabytes to bytes + const double gigaBytesToBytes = 1024. * 1024. * 1024.; + _maxMemory = static_cast(static_cast(memory) * gigaBytesToBytes); + _fetcher.updateCacheMemory(_maxMemory); } -void SequenceCache::setFetchingSequence(bool fetching) +QVariantList SequenceCache::getCachedFrames() const { - _fetchingSequence = fetching; - abortPrefetching = !fetching; - Q_EMIT requestHandled(); + return _fetcher.getCachedFrames(); } -void SequenceCache::setMemoryLimit(int memory) +void SequenceCache::setAsyncFetching(bool fetching) { - const double factorConvertGiB = 1024. * 1024. * 1024.; - const float fMemory = static_cast(memory * factorConvertGiB); - _cache = new aliceVision::image::ImageCache(fMemory, fMemory, aliceVision::image::EImageColorSpace::LINEAR); + // Always stop first + _fetcher.stopAsync(); + _threadPool.waitForDone(); + + if (fetching) + { + connect(&_fetcher, &AsyncFetcher::onAsyncFetchProgressed, this, &SequenceCache::onAsyncFetchProgressed); + _threadPool.start(&_fetcher); + } } QPointF SequenceCache::getRamInfo() const { - // get available RAM in bytes and cache occupied memory + // Get available RAM in bytes and cache occupied memory const auto memInfo = aliceVision::system::getMemoryInfo(); - // return in GB - return QPointF(static_cast(memInfo.availableRam / (1024. * 1024. * 1024.)), _cache->info().contentSize / (1024. * 1024. * 1024. * 1024.)); + + double availableRam = static_cast(memInfo.availableRam) / (1024. * 1024. * 1024.); + double contentSize = static_cast(_fetcher.getCacheSize()) / (1024. * 1024. * 1024. * 1024.); + + // Return in GB + return QPointF(availableRam, contentSize); } ResponseData SequenceCache::request(const RequestData& reqData) @@ -227,277 +106,35 @@ ResponseData SequenceCache::request(const RequestData& reqData) // Initialize empty response ResponseData response; - // Retrieve frame number corresponding to the requested image in the sequence - int frame = getFrame(reqData.path); - if (frame < 0) - { - // Empty response - return response; - } - - // Retrieve frame data - const std::size_t idx = static_cast(frame); - const FrameData& data = _sequence[idx]; - - // Retrieve image from cache - const bool cachedOnly = true; - const bool lazyCleaning = false; - response.img = _cache->get(data.path, data.downscale, cachedOnly, lazyCleaning); + std::shared_ptr> image; + oiio::ParamValueList metadatas; + size_t originalWidth = 0; + size_t originalHeight = 0; - // Retrieve metadata - response.dim = data.dim; - response.metadata = data.metadata; - - // Requested image is not in cache - // and there is already a prefetching thread running - if (!response.img && _loading && _interactivePrefetching) + if (!_fetcher.getFrame(reqData.path, image, metadatas, originalWidth, originalHeight)) { - // Abort prefetching to avoid waiting until current worker thread is done - abortPrefetching = true; + return response; } - // If requested image is not in cache and prefetching is disabled - if (!response.img && !_fetchingSequence) - { - // Load image in cache - try - { - const bool cachedOnly = false; - const bool lazyCleaning = false; - response.img = _cache->get(data.path, data.downscale, cachedOnly, lazyCleaning); - } - catch (const std::runtime_error& e) - { - // Log error message - std::cerr << e.what() << std::endl; - } - } + response.metadata.clear(); + response.img = image; + response.dim = QSize(static_cast(originalWidth), static_cast(originalHeight)); - // Request falls outside of safe region and we only want to fetch what is forward and not before - if ((frame < _regionSafe.first || frame > _regionSafe.second) && !_loading && _fetchingSequence) + // Convert metadatas + for (const auto& item : metadatas) { - // Make sur abort flag is off before launching a new prefetching thread - abortPrefetching = false; - - // Update internal state - _loading = true; - - // Gather images to load - std::vector toLoad = _sequence; - - // For now fill the allow worker thread to fill the whole cache capacity - const double fillRatio = 1.; - - // Create new runnable and launch it in worker thread (managed by Qt thread pool) - auto ioRunnable = new PrefetchingIORunnable(_cache, toLoad, frame, fillRatio, _sequenceId.loadAcquire()); - connect(ioRunnable, &PrefetchingIORunnable::progressed, this, &SequenceCache::onPrefetchingProgressed); - connect(ioRunnable, &PrefetchingIORunnable::done, this, &SequenceCache::onPrefetchingDone); - _threadPool.start(ioRunnable); + response.metadata[QString::fromStdString(item.name().string())] = QString::fromStdString(item.get_string()); } return response; } -void SequenceCache::onPrefetchingProgressed(int) +void SequenceCache::onAsyncFetchProgressed() { // Notify listeners that cache content has changed Q_EMIT requestHandled(); } -void SequenceCache::onPrefetchingDone(int sequenceId, int reqFrame) -{ - // Make sure the fetching concerns the actual sequence Id - bool exitOld = false; - _lockSequence.lock(); - if (sequenceId != _sequenceId) - { - exitOld = true; - } - if (reqFrame >= _sequence.size()) - { - exitOld = true; - } - _loading = false; - _lockSequence.unlock(); - - if (exitOld) - { - Q_EMIT requestHandled(); - return; - } - - _lockSequence.lock(); - { - // Retrieve cached region around requested frame - auto regionCached = std::make_pair(-1, -1); - for (int frame = reqFrame; frame >= 0; --frame) - { - const std::size_t idx = static_cast(frame); - - // Grow region on the left as much as possible - if (_cache->contains(_sequence[idx].path, _sequence[idx].downscale)) - { - regionCached.first = frame; - } - else - { - break; - } - } - for (int frame = reqFrame; frame < static_cast(_sequence.size()); ++frame) - { - const std::size_t idx = static_cast(frame); - - // Grow region on the right as much as possible - if (_cache->contains(_sequence[idx].path, _sequence[idx].downscale)) - { - regionCached.second = frame; - } - else - { - break; - } - } - - // Update safe region - if (regionCached == std::make_pair(-1, -1)) - { - _regionSafe = std::make_pair(-1, -1); - } - else - { - // Here we define safe region to cover 80% of cached region - // The remaining 20% serves to anticipate prefetching - const int extentCached = (regionCached.second - regionCached.first) / 2; - const int extentSafe = static_cast(static_cast(extentCached) * 0.8); - _regionSafe = buildRegion(reqFrame, extentSafe); - } - } - _lockSequence.unlock(); - - // Notify clients that a request has been handled - Q_EMIT requestHandled(); -} - -int SequenceCache::getFrame(const std::string& path) const -{ - // Go through frames until we find a matching filepath - for (int idx = 0; idx < _sequence.size(); ++idx) - { - if (_sequence[idx].path == path) - { - return idx; - } - } - - // No match found - return -1; -} - -std::pair SequenceCache::buildRegion(int frame, int extent) const -{ - // Initialize region equally around central frame - int start = frame - extent; - int end = frame + extent; - - // Adjust to sequence bounds - if (start < 0) - { - start = 0; - end = std::min(static_cast(_sequence.size()) - 1, 2 * extent); - } - else if (end >= static_cast(_sequence.size())) - { - end = static_cast(_sequence.size()) - 1; - start = std::max(0, static_cast(_sequence.size()) - 1 - 2 * extent); - } - - return std::make_pair(start, end); -} - -PrefetchingIORunnable::PrefetchingIORunnable(aliceVision::image::ImageCache* cache, - const std::vector& toLoad, - int reqFrame, - double fillRatio, - int sequenceId) - : _cache(cache), - _toLoad(toLoad), - _reqFrame(reqFrame), - _sequenceId(sequenceId) -{ - _toFill = static_cast(static_cast(_cache->info().capacity) * fillRatio); -} - -PrefetchingIORunnable::~PrefetchingIORunnable() {} - -void PrefetchingIORunnable::run() -{ - using namespace std::chrono_literals; - - // Timer for sending progress signals - auto tRef = std::chrono::high_resolution_clock::now(); - - // Processing order: - // Take the frames that are after the requested frame and put the rest after - std::vector toLoad; - toLoad.reserve(_toLoad.size()); - toLoad.insert(toLoad.end(), _toLoad.begin() + _reqFrame, _toLoad.end()); - if (_reqFrame > 0) - toLoad.insert(toLoad.end(), _toLoad.begin(), _toLoad.begin() + _reqFrame - 1); - _toLoad = toLoad; - - // Accumulator variable to keep track of cache capacity filled with loaded images - uint64_t filled = 0; - - // Load images from disk to cache - for (const auto& data : _toLoad) - { - // Check if main thread wants to abort prefetching - if (abortPrefetching) - { - abortPrefetching = false; - Q_EMIT done(_sequenceId, _reqFrame); - return; - } - - // Check if image size does not exceed limit - uint64_t memSize = static_cast(data.dim.width() / data.downscale) * static_cast(data.dim.height() / data.downscale) * 16; - if (filled + memSize > _toFill) - { - break; - } - - // Load image in cache - try - { - const bool cachedOnly = false; - const bool lazyCleaning = false; - _cache->get(data.path, data.downscale, cachedOnly, lazyCleaning); - filled += memSize; - } - catch (const std::runtime_error& e) - { - // Log error message - std::cerr << e.what() << std::endl; - } - - // Wait a few milliseconds in case another thread needs to query the cache - std::this_thread::sleep_for(1ms); - - // Regularly send progress signals - auto tNow = std::chrono::high_resolution_clock::now(); - std::chrono::duration diff = tNow - tRef; - if (diff.count() > 1.) - { - tRef = tNow; - Q_EMIT progressed(_reqFrame); - } - } - - // Notify main thread that loading is done - Q_EMIT done(_sequenceId, _reqFrame); -} - } // namespace imgserve } // namespace qtAliceVision diff --git a/src/qtAliceVision/SequenceCache.hpp b/src/qtAliceVision/SequenceCache.hpp index 987995cb..9abe224b 100644 --- a/src/qtAliceVision/SequenceCache.hpp +++ b/src/qtAliceVision/SequenceCache.hpp @@ -1,56 +1,16 @@ #pragma once #include "ImageServer.hpp" - -#include +#include "ImageCache.hpp" +#include "AsyncFetcher.hpp" #include -#include -#include -#include -#include -#include +#include #include -#include -#include - -#include -#include -#include -#include -#include namespace qtAliceVision { namespace imgserve { -/** - * @brief Utility struct for manipulating various information about a given frame. - */ -struct FrameData -{ - std::string path; - - QSize dim; - - QVariantMap metadata; - - int frame; - - int downscale; -}; - -/** - * @brief Image server with a caching system for loading image sequences from disk. - * - * Given a sequence of images (ordered by filename), the SequenceCache works as an image server: - * it receives requests from clients (in the form of a filepath) - * and its purpose is to provide the corresponding images (if they exist in the sequence). - * - * The SequenceCache takes advantage of the ordering of the sequence to load whole "regions" at once - * (a region being a contiguous range of images from the sequence). - * Such strategy makes sense under the assumption that the sequence order is meaningful for clients, - * i.e. that if an image is queried then it is likely that the next queries will be close in the sequence. - */ class SequenceCache : public QObject, public ImageServer { Q_OBJECT @@ -73,16 +33,10 @@ class SequenceCache : public QObject, public ImageServer void setSequence(const QVariantList& paths); /** - * @brief Toggle on/off interactive prefetching. - * @param[in] interactive new value for interactive prefetching flag - */ - void setInteractivePrefetching(bool interactive); - - /** - * @brief Set the target size for the images in the sequence. - * @param[in] size target size + * @brief Set the resize ratio for the images in the sequence. + * @param[in] ratio target ratio for the image downscale */ - void setTargetSize(int size); + void setResizeRatio(double ratio); /** * @brief Get the frames in the sequence that are currently cached. @@ -92,10 +46,10 @@ class SequenceCache : public QObject, public ImageServer QVariantList getCachedFrames() const; /** - * @brief Set the boolean flag indicating if the sequence is being fetched. + * @brief Set the boolean flag indicating if the sequence is being fetched asynchronously. * @param[in] fetching new value for the fetching flag */ - void setFetchingSequence(bool fetching); + void setAsyncFetching(bool fetching); /** * @brief Set the maximum memory that can be filled by the cache. @@ -116,135 +70,14 @@ class SequenceCache : public QObject, public ImageServer /// this method will launch a worker thread to prefetch new images from disk. ResponseData request(const RequestData& reqData) override; - /** - * @brief Slot called every time the prefetching thread progressed. - * @param[in] reqFrame the frame initially requested when the worker thread was started - */ - Q_SLOT void onPrefetchingProgressed(int reqFrame); - - /** - * @brief Slot called when the prefetching thread is finished. - * @param[in] sequenceId the sequenceId initially used when the worker thread was started - * @param[in] reqFrame the frame initially requested when the worker thread was started - */ - Q_SLOT void onPrefetchingDone(int sequenceId, int reqFrame); - - /** - * @brief Signal emitted when the prefetching thread is done and a previous request has been handled. - */ + public: + Q_SLOT void onAsyncFetchProgressed(); Q_SIGNAL void requestHandled(); - /** - * @brief Signal emitted when sequence content has been modified. - */ - Q_SIGNAL void contentChanged(); - private: - // Member variables - - /// Ordered sequence of frames. - std::vector _sequence; - - /// Image cache. - aliceVision::image::ImageCache* _cache; - - /// Frame interval used to decide if a prefetching thread should be launched. - std::pair _regionSafe; - - /// Keep track of whether or not there is an active worker thread. - bool _loading; - - /// Allow main thread to abort the prefetching thread and restart a centered around a more accurate location - bool _interactivePrefetching; - - /// Target size used to compute downscale - int _targetSize; - - /// Flag to indicate if the sequence is being fetched - bool _fetchingSequence; - - /// Local threadpool + size_t _maxMemory; + AsyncFetcher _fetcher; QThreadPool _threadPool; - - /// Current sequence id - QAtomicInt _sequenceId; - - /// sequence mutex - QMutex _lockSequence; - - private: - // Utility methods - - /** - * @brief Retrieve frame number corresponding to an image in the sequence. - * @param[in] path filepath of an image in the sequence - * @return frame number of the queried image if it is in the sequence, otherwise -1 - */ - int getFrame(const std::string& path) const; - - /** - * @brief Build a frame interval in the sequence. - * @param[in] frame central frame of the interval - * @param[in] extent interval half-size - * @return an interval of size 2*extent that fits in the sequence and contains the given frame - */ - std::pair buildRegion(int frame, int extent) const; -}; - -/** - * @brief Utility class for loading images from disk to cache asynchronously. - */ -class PrefetchingIORunnable : public QObject, public QRunnable -{ - Q_OBJECT - - public: - /** - * @param[in] cache pointer to image cache to fill - * @param[in] toLoad sequence frames to load from disk - * @param[in] reqFrame initially requested frame - * @param[in] fillRatio proportion of cache capacity that can be filled - * @param[in] sequenceId sequenceId to memorize - */ - PrefetchingIORunnable(aliceVision::image::ImageCache* cache, - const std::vector& toLoad, - int reqFrame, - double fillRatio, - int sequenceId); - - ~PrefetchingIORunnable(); - - /// Main method for loading images from disk to cache in a worker thread. - Q_SLOT void run() override; - - /** - * @brief Signal emitted regularly during prefetching when progress is made. - * @param[in] reqFrame initially requested frame - */ - Q_SIGNAL void progressed(int reqFrame); - - /** - * @brief Signal emitted when prefetching is finished. - * @param[in] sequenceId sequenceId at the time of launch - * @param[in] reqFrame initially requested frame - */ - Q_SIGNAL void done(int sequenceId, int reqFrame); - - private: - /// Image cache to fill. - aliceVision::image::ImageCache* _cache; - - /// Frames to load in cache. - std::vector _toLoad; - - /// Initially requested frame, used as central point for loading order. - int _reqFrame; - - /// Maximum memory that can be filled. - uint64_t _toFill; - - /// Sequence id - int _sequenceId; }; } // namespace imgserve