Add multithreaded upload to dynamic texture atlas on Vulkan

include/mbgl/gfx/dynamic_texture.hpp

@@ -4,6 +4,8 @@
 #include <mbgl/util/image.hpp>
 #include <mbgl/util/rect.hpp>
 
+#include <mbgl/vulkan/texture2d.hpp>
+
 #include <mapbox/shelf-pack.hpp>
 
 #include <optional>
@@ -53,13 +55,23 @@ class DynamicTexture {
     bool isEmpty() const;
 
     std::optional<TextureHandle> reserveSize(const Size& size, int32_t uniqueId);
-    void uploadImage(const uint8_t* pixelData, TextureHandle& texHandle);
+    void uploadImage(const uint8_t* pixelData,
+                     TextureHandle& texHandle,
+                     std::vector<std::function<void(Context&)>>& deletionQueue,
+                     const vk::UniqueCommandBuffer& commandBuffer);
 
     template <typename Image>
-    std::optional<TextureHandle> addImage(const Image& image, int32_t uniqueId = -1) {
-        return addImage(image.data ? image.data.get() : nullptr, image.size, uniqueId);
+    std::optional<TextureHandle> addImage(const Image& image,
+                                          std::vector<std::function<void(Context&)>>& deletionQueue,
+                                          const vk::UniqueCommandBuffer& commandBuffer,
+                                          int32_t uniqueId = -1) {
+        return addImage(image.data ? image.data.get() : nullptr, image.size, deletionQueue, commandBuffer, uniqueId);
     }
-    std::optional<TextureHandle> addImage(const uint8_t* pixelData, const Size& imageSize, int32_t uniqueId = -1);
+    std::optional<TextureHandle> addImage(const uint8_t* pixelData,
+                                          const Size& imageSize,
+                                          std::vector<std::function<void(Context&)>>& deletionQueue,
+                                          const vk::UniqueCommandBuffer& commandBuffer,
+                                          int32_t uniqueId = -1);
 
     void uploadDeferredImages();
     void removeTexture(const TextureHandle& texHandle);
@@ -75,8 +87,7 @@ class DynamicTexture {
     std::mutex mutex;
 };
 
-#define MLN_DEFER_UPLOAD_ON_RENDER_THREAD \
-    (MLN_RENDER_BACKEND_OPENGL || MLN_RENDER_BACKEND_VULKAN || MLN_RENDER_BACKEND_WEBGPU)
+#define MLN_DEFER_UPLOAD_ON_RENDER_THREAD (MLN_RENDER_BACKEND_OPENGL || MLN_RENDER_BACKEND_WEBGPU)
 
 } // namespace gfx
 } // namespace mbgl

include/mbgl/gfx/dynamic_texture_atlas.hpp

@@ -31,15 +31,20 @@ class DynamicTextureAtlas {
         : context(context_) {}
     ~DynamicTextureAtlas() = default;
 
-    GlyphAtlas uploadGlyphs(const GlyphMap& glyphs);
+    GlyphAtlas uploadGlyphs(const GlyphMap& glyphs,
+                            std::vector<std::function<void(Context&)>>& deletionQueue,
+                            const vk::UniqueCommandBuffer& commandBuffer);
     ImageAtlas uploadIconsAndPatterns(const ImageMap& icons,
                                       const ImageMap& patterns,
-                                      const ImageVersionMap& versionMap);
+                                      const ImageVersionMap& versionMap,
+                                      std::vector<std::function<void(Context&)>>& deletionQueue,
+                                      const vk::UniqueCommandBuffer& commandBuffer);
 
     void removeTextures(const std::vector<TextureHandle>& textureHandles, const DynamicTexturePtr& dynamicTexture);
 
-private:
     Context& context;
+
+private:
     std::vector<DynamicTexturePtr> dynamicTextures;
     std::unordered_map<TexturePixelType, DynamicTexturePtr> dummyDynamicTexture;
     std::mutex mutex;

include/mbgl/vulkan/context.hpp

@@ -144,7 +144,8 @@ class Context final : public gfx::Context {
 
     uint8_t getCurrentFrameResourceIndex() const { return frameResourceIndex; }
     void enqueueDeletion(std::function<void(Context&)>&& function);
-    void submitOneTimeCommand(const std::function<void(const vk::UniqueCommandBuffer&)>& function) const;
+    void submitOneTimeCommand(vk::UniqueCommandPool* commandPool,
+                              const std::function<void(const vk::UniqueCommandBuffer&)>& function) const;
 
     void requestSurfaceUpdate(bool useDelay = true);
 

include/mbgl/vulkan/texture2d.hpp

@@ -72,6 +72,7 @@ class Texture2D : public gfx::Texture2D {
                          const Size& size,
                          uint16_t xOffset,
                          uint16_t yOffset,
+                         std::vector<std::function<void(gfx::Context&)>>* deletionQueue,
                          const vk::UniqueCommandBuffer& buffer) noexcept;
 
     bool needsUpload() const noexcept override { return !!imageData; };

src/mbgl/gfx/dynamic_texture.cpp

@@ -2,6 +2,8 @@
 #include <mbgl/gfx/texture2d.hpp>
 #include <mbgl/gfx/context.hpp>
 
+#include <mbgl/vulkan/texture2d.hpp>
+
 namespace mbgl {
 namespace gfx {
 
@@ -48,7 +50,10 @@ std::optional<TextureHandle> DynamicTexture::reserveSize(const Size& size, int32
     return TextureHandle(*bin);
 }
 
-void DynamicTexture::uploadImage(const uint8_t* pixelData, TextureHandle& texHandle) {
+void DynamicTexture::uploadImage(const uint8_t* pixelData,
+                                 TextureHandle& texHandle,
+                                 std::vector<std::function<void(Context&)>>& deletionQueue,
+                                 const vk::UniqueCommandBuffer& commandBuffer) {
     std::lock_guard<std::mutex> lock(mutex);
     const auto& rect = texHandle.getRectangle();
     const auto imageSize = Size(rect.w, rect.h);
@@ -59,17 +64,20 @@ void DynamicTexture::uploadImage(const uint8_t* pixelData, TextureHandle& texHan
     std::copy(pixelData, pixelData + size, imageData.get());
     imagesToUpload.emplace(texHandle, std::move(imageData));
 #else
-    texture->uploadSubRegion(pixelData, imageSize, rect.x, rect.y);
+    static_cast<vulkan::Texture2D*>(texture.get())
+        ->uploadSubRegion(pixelData, imageSize, rect.x, rect.y, &deletionQueue, commandBuffer);
 #endif
     texHandle.needsUpload = false;
 }
 
 std::optional<TextureHandle> DynamicTexture::addImage(const uint8_t* pixelData,
                                                       const Size& imageSize,
+                                                      std::vector<std::function<void(Context&)>>& deletionQueue,
+                                                      const vk::UniqueCommandBuffer& commandBuffer,
                                                       int32_t uniqueId) {
     auto texHandle = reserveSize(imageSize, uniqueId);
     if (texHandle && texHandle->isUploadNeeded()) {
-        uploadImage(pixelData, *texHandle);
+        uploadImage(pixelData, *texHandle, deletionQueue, commandBuffer);
     }
     return texHandle;
 }

src/mbgl/gfx/dynamic_texture_atlas.cpp

@@ -16,7 +16,9 @@ Rect<uint16_t> rectWithoutExtraPadding(const Rect<uint16_t>& rect) {
         rect.x + extraPadding, rect.y + extraPadding, rect.w - 2 * extraPadding, rect.h - 2 * extraPadding);
 }
 
-GlyphAtlas DynamicTextureAtlas::uploadGlyphs(const GlyphMap& glyphs) {
+GlyphAtlas DynamicTextureAtlas::uploadGlyphs(const GlyphMap& glyphs,
+                                             std::vector<std::function<void(Context&)>>& deletionQueue,
+                                             const vk::UniqueCommandBuffer& commandBuffer) {
     using GlyphsToUpload = std::vector<std::tuple<TextureHandle, Immutable<Glyph>, FontStackHash>>;
     std::lock_guard<std::mutex> lock(mutex);
 
@@ -28,7 +30,7 @@ GlyphAtlas DynamicTextureAtlas::uploadGlyphs(const GlyphMap& glyphs) {
             dummyImage.fill(0);
             glyphAtlas.dynamicTexture = std::make_shared<gfx::DynamicTexture>(
                 context, dummySize, TexturePixelType::Alpha);
-            glyphAtlas.dynamicTexture->addImage(dummyImage.data.get(), dummySize);
+            glyphAtlas.dynamicTexture->addImage(dummyImage.data.get(), dummySize, deletionQueue, commandBuffer);
             dummyDynamicTexture[TexturePixelType::Alpha] = glyphAtlas.dynamicTexture;
         }
         return glyphAtlas;
@@ -94,7 +96,7 @@ GlyphAtlas DynamicTextureAtlas::uploadGlyphs(const GlyphMap& glyphs) {
             paddedImage.fill(0);
             AlphaImage::copy(glyph->bitmap, paddedImage, {0, 0}, {padding, padding}, glyph->bitmap.size);
 
-            glyphAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle);
+            glyphAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle, deletionQueue, commandBuffer);
         }
         glyphAtlas.textureHandles.emplace_back(texHandle);
         glyphAtlas.glyphPositions[fontStack].emplace(glyph->id,
@@ -105,7 +107,9 @@ GlyphAtlas DynamicTextureAtlas::uploadGlyphs(const GlyphMap& glyphs) {
 
 ImageAtlas DynamicTextureAtlas::uploadIconsAndPatterns(const ImageMap& icons,
                                                        const ImageMap& patterns,
-                                                       const ImageVersionMap& versionMap) {
+                                                       const ImageVersionMap& versionMap,
+                                                       std::vector<std::function<void(Context&)>>& deletionQueue,
+                                                       const vk::UniqueCommandBuffer& commandBuffer) {
     using ImagesToUpload = std::vector<std::pair<TextureHandle, Immutable<style::Image::Impl>>>;
     std::lock_guard<std::mutex> lock(mutex);
 
@@ -117,7 +121,7 @@ ImageAtlas DynamicTextureAtlas::uploadIconsAndPatterns(const ImageMap& icons,
             dummyImage.fill(0);
             imageAtlas.dynamicTexture = std::make_shared<gfx::DynamicTexture>(
                 context, dummySize, TexturePixelType::RGBA);
-            imageAtlas.dynamicTexture->addImage(dummyImage.data.get(), dummySize);
+            imageAtlas.dynamicTexture->addImage(dummyImage.data.get(), dummySize, deletionQueue, commandBuffer);
             dummyDynamicTexture[TexturePixelType::RGBA] = imageAtlas.dynamicTexture;
         }
         return imageAtlas;
@@ -201,7 +205,7 @@ ImageAtlas DynamicTextureAtlas::uploadIconsAndPatterns(const ImageMap& icons,
             paddedImage.fill(0);
             PremultipliedImage::copy(icon->image, paddedImage, {0, 0}, {padding, padding}, icon->image.size);
 
-            imageAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle);
+            imageAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle, deletionQueue, commandBuffer);
         }
         imageAtlas.textureHandles.emplace_back(texHandle);
         const auto it = versionMap.find(icon->id);
@@ -229,7 +233,7 @@ ImageAtlas DynamicTextureAtlas::uploadIconsAndPatterns(const ImageMap& icons,
             PremultipliedImage::copy(pattern->image, paddedImage, {w - 1, 0}, {x - 1, y}, {1, h}); // L
             PremultipliedImage::copy(pattern->image, paddedImage, {0, 0}, {x + w, y}, {1, h});     // R
 
-            imageAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle);
+            imageAtlas.dynamicTexture->uploadImage(paddedImage.data.get(), texHandle, deletionQueue, commandBuffer);
         }
         imageAtlas.textureHandles.emplace_back(texHandle);
         const auto it = versionMap.find(pattern->id);

src/mbgl/tile/geometry_tile_worker.cpp

@@ -22,6 +22,8 @@
 #include <mbgl/util/stopwatch.hpp>
 #include <mbgl/util/thread_pool.hpp>
 
+#include <mbgl/vulkan/context.hpp>
+
 #include <unordered_set>
 #include <utility>
 
@@ -50,11 +52,18 @@ GeometryTileWorker::GeometryTileWorker(OptionalActorRef<GeometryTileWorker> self
       pixelRatio(pixelRatio_),
       showCollisionBoxes(showCollisionBoxes_),
       dynamicTextureAtlas(dynamicTextureAtlas_),
-      fontFaces(fontFaces_) {}
+      fontFaces(fontFaces_) {
+    const auto& contextVK = static_cast<vulkan::Context&>(dynamicTextureAtlas->context);
+    const vk::CommandPoolCreateInfo createInfo(vk::CommandPoolCreateFlagBits::eResetCommandBuffer,
+                                               contextVK.getBackend().getGraphicsQueueIndex());
+    commandPool = contextVK.getBackend().getDevice()->createCommandPoolUnique(
+        createInfo, nullptr, contextVK.getBackend().getDispatcher());
+}
 
 GeometryTileWorker::~GeometryTileWorker() {
     MLN_TRACE_FUNC();
 
+    commandPool.reset();
     scheduler.runOnRenderThread([renderData_{std::move(renderData)}]() {});
 }
 
@@ -542,13 +551,18 @@ void GeometryTileWorker::finalizeLayout() {
     gfx::ImageAtlas imageAtlas;
     gfx::GlyphAtlas glyphAtlas;
     if (dynamicTextureAtlas) {
-        imageAtlas = dynamicTextureAtlas->uploadIconsAndPatterns(iconMap, patternMap, versionMap);
+        std::vector<std::function<void(gfx::Context&)>> deletionQueue;
+        const auto& contextVK = static_cast<vulkan::Context&>(dynamicTextureAtlas->context);
+        contextVK.submitOneTimeCommand(&commandPool, [&](const vk::UniqueCommandBuffer& commandBuffer) {
+            imageAtlas = dynamicTextureAtlas->uploadIconsAndPatterns(
+                iconMap, patternMap, versionMap, deletionQueue, commandBuffer);
+            glyphAtlas = dynamicTextureAtlas->uploadGlyphs(glyphMap, deletionQueue, commandBuffer);
+        });
+        for (const auto& function : deletionQueue) function(dynamicTextureAtlas->context);
+        deletionQueue.clear();
     }
-    if (!layouts.empty()) {
-        if (dynamicTextureAtlas) {
-            glyphAtlas = dynamicTextureAtlas->uploadGlyphs(glyphMap);
-        }
 
+    if (!layouts.empty()) {
         for (auto& layout : layouts) {
             if (obsolete) {
                 dynamicTextureAtlas->removeTextures(glyphAtlas.textureHandles, glyphAtlas.dynamicTexture);

src/mbgl/tile/geometry_tile_worker.hpp

@@ -17,6 +17,8 @@
 #include <atomic>
 #include <memory>
 
+#include <mbgl/vulkan/context.hpp>
+
 namespace mbgl {
 
 class GeometryTile;
@@ -121,6 +123,7 @@ class GeometryTileWorker {
     bool firstLoad = true;
 
     gfx::DynamicTextureAtlasPtr dynamicTextureAtlas;
+    vk::UniqueCommandPool commandPool;
 
     std::shared_ptr<FontFaces> fontFaces;
 };

src/mbgl/vulkan/context.cpp

@@ -167,11 +167,21 @@ void Context::enqueueDeletion(std::function<void(Context&)>&& function) {
     frameResources[frameResourceIndex].deletionQueue.push_back(std::move(function));
 }
 
-void Context::submitOneTimeCommand(const std::function<void(const vk::UniqueCommandBuffer&)>& function) const {
+std::mutex mutex;
+void Context::submitOneTimeCommand(vk::UniqueCommandPool* commandPool,
+                                   const std::function<void(const vk::UniqueCommandBuffer&)>& function) const {
     MLN_TRACE_FUNC();
+    std::lock_guard<std::mutex> lock(mutex);
+
+    vk::UniqueCommandPool commandPoolLocal;
+    if (commandPool) {
+        const vk::CommandPoolCreateInfo createInfo(vk::CommandPoolCreateFlagBits::eResetCommandBuffer,
+                                                   backend.getGraphicsQueueIndex());
+        commandPoolLocal = backend.getDevice()->createCommandPoolUnique(createInfo, nullptr, backend.getDispatcher());
+    }
 
     const vk::CommandBufferAllocateInfo allocateInfo(
-        backend.getCommandPool().get(), vk::CommandBufferLevel::ePrimary, 1);
+        commandPool ? commandPoolLocal.get() : backend.getCommandPool().get(), vk::CommandBufferLevel::ePrimary, 1);
 
     const auto& device = backend.getDevice();
     const auto& dispatcher = backend.getDispatcher();
@@ -315,6 +325,8 @@ void Context::endFrame() {}
 
 void Context::submitFrame() {
     MLN_TRACE_FUNC();
+    std::lock_guard<std::mutex> lock(mutex);
+
     const auto& dispatcher = backend.getDispatcher();
     const auto& frame = frameResources[frameResourceIndex];
     frame.commandBuffer->end(dispatcher);
@@ -616,8 +628,8 @@ const std::unique_ptr<Texture2D>& Context::getDummyTexture() {
         dummyTexture2D->setFormat(gfx::TexturePixelType::RGBA, gfx::TextureChannelDataType::UnsignedByte);
         dummyTexture2D->setSize(size);
 
-        submitOneTimeCommand([&](const vk::UniqueCommandBuffer& commandBuffer) {
-            dummyTexture2D->uploadSubRegion(data.data(), size, 0, 0, commandBuffer);
+        submitOneTimeCommand(nullptr, [&](const vk::UniqueCommandBuffer& commandBuffer) {
+            dummyTexture2D->uploadSubRegion(data.data(), size, 0, 0, nullptr, commandBuffer);
         });
     }
 

src/mbgl/vulkan/texture2d.cpp

@@ -145,14 +145,15 @@ void Texture2D::uploadSubRegion(const void* pixelData, const Size& size_, uint16
     const auto& encoder = context.createCommandEncoder();
     const auto& encoderImpl = static_cast<const CommandEncoder&>(*encoder);
 
-    uploadSubRegion(pixelData, size_, xOffset, yOffset, encoderImpl.getCommandBuffer());
+    uploadSubRegion(pixelData, size_, xOffset, yOffset, nullptr, encoderImpl.getCommandBuffer());
 }
 
 void Texture2D::uploadSubRegion(const void* pixelData,
                                 const Size& size_,
                                 uint16_t xOffset,
                                 uint16_t yOffset,
-                                const vk::UniqueCommandBuffer& commandBuffer) noexcept {
+                                std::vector<std::function<void(gfx::Context&)>>* deletionQueue,
+                                [[maybe_unused]] const vk::UniqueCommandBuffer& commandBuffer) noexcept {
     if (!pixelData || size_.width == 0 || size_.height == 0) return;
 
     create();
@@ -203,9 +204,19 @@ void Texture2D::uploadSubRegion(const void* pixelData,
         }
     };
 
+    // context.submitOneTimeCommand([&](const vk::UniqueCommandBuffer& commandBuffer) {
     enqueueCommands(commandBuffer);
+    //});
 
-    context.enqueueDeletion([buffAlloc = std::move(bufferAllocation)](auto&) mutable { buffAlloc.reset(); });
+    const auto function = [buffAlloc = std::move(bufferAllocation)](auto&) mutable {
+        buffAlloc.reset();
+    };
+
+    if (deletionQueue) {
+        deletionQueue->push_back(std::move(function));
+    } else {
+        context.enqueueDeletion(std::move(function));
+    }
 
     context.renderingStats().numTextureUpdates++;
     context.renderingStats().textureUpdateBytes += bufferInfo.size;
@@ -500,7 +511,7 @@ void Texture2D::copyImage(vk::Image image) {
 
     create();
 
-    context.submitOneTimeCommand([&](const vk::UniqueCommandBuffer& commandBuffer) {
+    context.submitOneTimeCommand(nullptr, [&](const vk::UniqueCommandBuffer& commandBuffer) {
         const auto copyInfo = vk::ImageCopy()
                                   .setSrcSubresource({vk::ImageAspectFlagBits::eColor, 0, 0, 1})
                                   .setDstSubresource({vk::ImageAspectFlagBits::eColor, 0, 0, 1})
@@ -553,7 +564,7 @@ std::shared_ptr<PremultipliedImage> Texture2D::readImage() {
         }
 
         // Copy image to staging buffer
-        context.submitOneTimeCommand([&](const vk::UniqueCommandBuffer& commandBuffer) {
+        context.submitOneTimeCommand(nullptr, [&](const vk::UniqueCommandBuffer& commandBuffer) {
             // Transition image layout for reading
             const auto barrier = vk::ImageMemoryBarrier()
                                      .setImage(imageAllocation->image)