JBR-7766 Fix VKTexturePool OOM.

Author: YaaZ

src/java.desktop/share/native/common/java2d/vulkan/VKAllocator.c

@@ -191,6 +191,14 @@ void VKAllocator_FindMemoryType(VKMemoryRequirements* requirements,
 static uint32_t VKAllocator_AllocatePage(VKAllocator* alloc, uint32_t memoryType, VkDeviceSize size,
                                          VkImage dedicatedImage, VkBuffer dedicatedBuffer) {
     assert(alloc != NULL);
+    assert(memoryType < VK_MAX_MEMORY_TYPES);
+
+    uint32_t heapIndex = alloc->memoryProperties.memoryTypes[memoryType].heapIndex;
+    VkDeviceSize heapSize = alloc->memoryProperties.memoryHeaps[heapIndex].size;
+    if (size > heapSize) {
+        J2dRlsTraceLn3(J2D_TRACE_ERROR, "VKAllocator_AllocatePage: not enough memory in heap, heapIndex=%d, heapSize=%d, size=%d", heapIndex, heapSize, size);
+        return NO_PAGE_INDEX;
+    }
 
     // Allocate memory.
     VkBool32 dedicated = dedicatedImage != VK_NULL_HANDLE || dedicatedBuffer != VK_NULL_HANDLE;

src/java.desktop/share/native/common/java2d/vulkan/VKTexturePool.c

@@ -73,8 +73,7 @@ void VKTexturePoolLock_unlockImpl(ATexturePoolLockPrivPtr *lock) {
 }
 
 static void VKTexturePool_FindImageMemoryType(VKMemoryRequirements* requirements) {
-    // TODO both DEVICE_LOCAL and HOST_VISIBLE memory is very precious, we may need to use just DEVICE_LOCAL instead.
-    VKAllocator_FindMemoryType(requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, VK_ALL_MEMORY_PROPERTIES);
+    VKAllocator_FindMemoryType(requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_ALL_MEMORY_PROPERTIES);
 }
 
 /* Texture allocate/free API */