feat: include VML itself in layer lookup. (#20999)

Author: rabbitism

src/Avalonia.Controls/Primitives/AdornerLayer.cs

@@ -73,7 +73,7 @@ public static void SetAdornedElement(Visual adorner, Visual? adorned)
         public static AdornerLayer? GetAdornerLayer(Visual visual)
         {
             // Check if the visual is inside an OverlayLayer with a dedicated AdornerLayer
-            foreach (var ancestor in visual.GetVisualAncestors())
+            foreach (var ancestor in visual.GetSelfAndVisualAncestors())
             {
                 if (GetDirectAdornerLayer(ancestor) is { } adornerLayer)
                     return adornerLayer;

src/Avalonia.Controls/Primitives/OverlayLayer.cs

@@ -29,7 +29,7 @@ internal OverlayLayer()
         /// <returns>The <see cref="OverlayLayer"/> associated with the visual, or null if no overlay layer exists.</returns>
         public static OverlayLayer? GetOverlayLayer(Visual visual)
         {
-            foreach (var v in visual.GetVisualAncestors())
+            foreach (var v in visual.GetSelfAndVisualAncestors())
                 if (v is VisualLayerManager { OverlayLayer: { } layer })
                     return layer;
 

src/Avalonia.Controls/Primitives/PopupOverlayLayer.cs

@@ -11,7 +11,7 @@ internal sealed class PopupOverlayLayer : Canvas
 
         public static PopupOverlayLayer? GetPopupOverlayLayer(Visual visual)
         {
-            foreach (var v in visual.GetVisualAncestors())
+            foreach (var v in visual.GetSelfAndVisualAncestors())
                 if (v is VisualLayerManager { PopupOverlayLayer: { } layer })
                     return layer;
 

src/Avalonia.Controls/Primitives/TextSelectorLayer.cs

@@ -11,7 +11,7 @@ public class TextSelectorLayer : Canvas
 
         public static TextSelectorLayer? GetTextSelectorLayer(Visual visual)
         {
-            foreach (var v in visual.GetVisualAncestors())
+            foreach (var v in visual.GetSelfAndVisualAncestors())
                 if (v is VisualLayerManager { TextSelectorLayer: { } textSelectorLayer })
                     return textSelectorLayer;
 

tests/Avalonia.Controls.UnitTests/Primitives/VisualLayerManagerTests.cs

@@ -33,5 +33,83 @@ public void GetAdornerLayer_Returns_Dedicated_AdornerLayer_For_Controls_Inside_O
             Assert.NotNull(mainAdornerLayer);
             Assert.NotSame(overlayAdornerLayer, mainAdornerLayer);
         }
+        
+        [Fact]
+        public void GetAdornerLayer_Returns_Same_AdornerLayer_For_VisualLayerManager()
+        {
+            var vlm = new VisualLayerManager();
+            var root = new TestRoot { Child = vlm };
+
+            root.Measure(new Size(100, 100));
+            root.Arrange(new Rect(0, 0, 100, 100));
+
+            var adornerLayer = vlm.AdornerLayer;
+            Assert.NotNull(adornerLayer);
+            
+            // The adorner layer for a control inside the OverlayLayer
+            // should be the dedicated one, not the main VLM adorner layer.
+            var target = AdornerLayer.GetAdornerLayer(vlm);
+            Assert.NotNull(target);
+            Assert.Same(adornerLayer, target);
+        }
+        
+        [Fact]
+        public void GetAdornerLayer_Returns_Same_AdornerLayer_For_Child()
+        {
+            var button = new Button();
+            var vlm = new VisualLayerManager() { Child = button };
+            var root = new TestRoot { Child = vlm };
+
+            root.Measure(new Size(100, 100));
+            root.Arrange(new Rect(0, 0, 100, 100));
+
+            var adornerLayer = vlm.AdornerLayer;
+            Assert.NotNull(adornerLayer);
+            
+            // The adorner layer for a control inside the OverlayLayer
+            // should be the dedicated one, not the main VLM adorner layer.
+            var target = AdornerLayer.GetAdornerLayer(button);
+            Assert.NotNull(target);
+            Assert.Same(adornerLayer, target);
+        }
+        
+        [Fact]
+        public void GetOverlayLayer_Returns_Same_OverlayLayer_For_VisualLayerManager()
+        {
+            var vlm = new VisualLayerManager() { EnableOverlayLayer = true };
+            var root = new TestRoot { Child = vlm };
+
+            root.Measure(new Size(100, 100));
+            root.Arrange(new Rect(0, 0, 100, 100));
+
+            var overlayLayer = vlm.OverlayLayer;
+            Assert.NotNull(overlayLayer);
+            
+            // The adorner layer for a control inside the OverlayLayer
+            // should be the dedicated one, not the main VLM adorner layer.
+            var target = OverlayLayer.GetOverlayLayer(vlm);
+            Assert.NotNull(target);
+            Assert.Same(overlayLayer, target);
+        }
+        
+        [Fact]
+        public void GetOverlayLayer_Returns_Same_OverlayLayer_For_Child()
+        {
+            var button = new Button();
+            var vlm = new VisualLayerManager() { EnableOverlayLayer = true, Child = button };
+            var root = new TestRoot { Child = vlm };
+
+            root.Measure(new Size(100, 100));
+            root.Arrange(new Rect(0, 0, 100, 100));
+
+            var overlayLayer = vlm.OverlayLayer;
+            Assert.NotNull(overlayLayer);
+            
+            // The adorner layer for a control inside the OverlayLayer
+            // should be the dedicated one, not the main VLM adorner layer.
+            var target = OverlayLayer.GetOverlayLayer(button);
+            Assert.NotNull(target);
+            Assert.Same(overlayLayer, target);
+        }
     }
 }