From bb05fc4cd8173044a3e2b5dd040e5a354e9d8d8f Mon Sep 17 00:00:00 2001 From: pntmass Date: Sat, 19 Oct 2019 10:53:27 -0400 Subject: [PATCH 1/8] adding overlay and logic to pull partial stroke from inking thread --- ScreenToGif/Controls/InkCanvasExtended.cs | 108 +++++++++++++++++++++- ScreenToGif/ImageUtil/ImageMethods.cs | 37 ++++++++ ScreenToGif/Windows/Board.xaml.cs | 7 +- 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/ScreenToGif/Controls/InkCanvasExtended.cs b/ScreenToGif/Controls/InkCanvasExtended.cs index a3e3e6ba..b03badd9 100644 --- a/ScreenToGif/Controls/InkCanvasExtended.cs +++ b/ScreenToGif/Controls/InkCanvasExtended.cs @@ -1,7 +1,12 @@ -using System.Windows; +using ScreenToGif.ImageUtil; +using System; +using System.Reflection; +using System.Windows; using System.Windows.Controls; using System.Windows.Ink; +using System.Windows.Input.StylusPlugIns; using System.Windows.Media; +using System.Windows.Threading; namespace ScreenToGif.Controls { @@ -11,6 +16,19 @@ namespace ScreenToGif.Controls /// public class InkCanvasExtended : InkCanvas { + public InkCanvasExtended() + : base() + { + // We add a child image to use as an inking overlay because we need + // to compose the inking thread's real-time data with the UI data + // that contains already-captured strokes. + InkingImage = new Image(); + this.Children.Add(InkingImage); + // Inking overlay should be in front, since it is a "newer" pre-stroke + // element and once its stroke is brought over to the UI thread it will + // be in front of previous strokes. + Canvas.SetZIndex(InkingImage, 99); + } /// /// Gets or set the eraser shape /// @@ -25,6 +43,93 @@ public class InkCanvasExtended : InkCanvas public static readonly DependencyProperty EraserShapeProperty = DependencyProperty.Register("EraserShape", typeof (StylusShape), typeof (InkCanvasExtended), new UIPropertyMetadata(new RectangleStylusShape(10, 10), OnEraserShapePropertyChanged)); + private Image InkingImage { get; } + + private double _prevHeight; + private double _prevWidth; + /// + /// Updates the inking overlay from the real-time inking thread + /// + /// + public void UpdateInkOverlay() + { + // Set the source to nothing at first so we don't see things in the wrong place + // momentarily when adjusting the margins later on. + // Also good for when returning when no inking exists, but could do immediately + // before the return in that case. + InkingImage.Source = new DrawingImage(new DrawingGroup()); + + // The following two HostVisuals are children of the _mainContainerVisual in this thread, + // and these won't normally get composed into the UI thread from the inking thread's + // element tree until a stroke is complete. + var myArgs = new[] + { + typeof(DynamicRenderer).GetField("_rawInkHostVisual1", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(this.DynamicRenderer ), + typeof(DynamicRenderer).GetField("_rawInkHostVisual2", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(this.DynamicRenderer ) + }; + + // We just need to get a handle on the _renderingThread (inking thread), and its ThreadDispatcher. + var lRenderingThread = typeof(DynamicRenderer).GetField("_renderingThread", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(this.DynamicRenderer); + if(lRenderingThread == null) + { + return; + } + PropertyInfo lDynamicDispProperty = lRenderingThread.GetType().GetProperty("ThreadDispatcher", BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public); + + // The ThreadDispatcher derives from System.Windows.Threading.Dispatcher, + // which is a little simpler than using more reflection to get the derived type. + System.Windows.Threading.Dispatcher lDispatcher = lDynamicDispProperty.GetValue(lRenderingThread, null) as System.Windows.Threading.Dispatcher; + if (lRenderingThread == null) + { + return; + } + + // We invoke the inking thread to get the visual targets from the real time host visuals, then + // use BFS to grab all the DrawingGroups as frozen into a new DrawingGroup, which we return + // as frozen to the UI thread. + DrawingGroup inkDrawingGroup = lDispatcher.Invoke(DispatcherPriority.Send, + (DispatcherOperationCallback)delegate (object args) + { + // We've got the field references from the other thread now, so we just get their + // VisualTarget properties, which is where we'll make a magical RootVisual call. + object[] lObjects = args as object[]; + PropertyInfo lVtProperty = lObjects[0].GetType().GetProperty("VisualTarget", BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public); + VisualTarget VisualTarget1 = lVtProperty.GetValue(lObjects[0], null) as VisualTarget; + VisualTarget VisualTarget2 = lVtProperty.GetValue(lObjects[1], null) as VisualTarget; + + // The RootVisual builds the visual when property get is called. We then + // all all its descendent DrawingGroups to this DrawingGroup that we are + // just using as a collection to return from the thread. + DrawingGroup drawingGroups = new DrawingGroup(); + VisualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); + VisualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); + return drawingGroups.GetAsFrozen(); + }, + myArgs) as DrawingGroup; + + + // This is a little jenky, but we need to set the image margins otherwise + // the drawing content will be cropped and aligned top-left despite inking + // very far from origin. Because we set image to an empty drawingImage + // earlier, we don't need to worry about things visibly jumping on screen. + // Important note: Negatives are okay. If we set X,Y = (0,0), then the + // DrawingGroup.Bounds.TopLeft = (-3,0) [because user inked off the canvas + // for instance], then the point starting at (-3,0) will be shifted to (0,0) + // and our entire drawing will be moved over by 3 pixels, so here we just + // set the margin to whatever *negative* number the TopLeft has in that case, + // and we care only about going over the actual width/height (e.g. infinity) + var bounds = inkDrawingGroup.Bounds.TopLeft; + var w = System.Math.Min(bounds.X, this.ActualWidth); + var h = System.Math.Min(bounds.Y, this.ActualHeight); + InkingImage.Margin = new System.Windows.Thickness(w, h, 0, 0); + + // At this point, just update the image source with the drawing image. + InkingImage.Source = new DrawingImage(inkDrawingGroup); + } /// /// Event to handle the property change /// @@ -39,4 +144,5 @@ private static void OnEraserShapePropertyChanged(DependencyObject d, DependencyP canvas.RenderTransform = new MatrixTransform(); } } + } \ No newline at end of file diff --git a/ScreenToGif/ImageUtil/ImageMethods.cs b/ScreenToGif/ImageUtil/ImageMethods.cs index bd31793c..76cc7ab7 100644 --- a/ScreenToGif/ImageUtil/ImageMethods.cs +++ b/ScreenToGif/ImageUtil/ImageMethods.cs @@ -1504,6 +1504,43 @@ public static Icon ToIcon(this ImageSource imageSource) } } + /// + /// Visits all the descendents of the visual using DFS and adds frozen copies + /// of their drawing objects as children to the drawingGroup argument. + /// + /// The visual to convert to a DrawingGroup + /// The target DrawingGroup to be populated + static public void visualToFrozenDrawingGroup(this Visual visual, DrawingGroup drawingGroup) + { + if (visual == null) + { + return; + } + Queue visualQueue = new Queue(new[] { visual }); + while (visualQueue.Count > 0) + { + Visual visualDescendent = visualQueue.Dequeue(); + for (int i = 0; i < VisualTreeHelper.GetChildrenCount(visualDescendent); i++) + { + visualQueue.Enqueue(VisualTreeHelper.GetChild(visualDescendent, i) as Visual); + } + DrawingGroup vdg = VisualTreeHelper.GetDrawing(visualDescendent); + if (vdg != null) + { + drawingGroup.Children.Add(vdg.GetAsFrozen() as DrawingGroup); + } + } + } + + /// + /// Gets a DrawingGroup with frozen copies of all the visual's descendent DrawingGroups + /// + static public DrawingGroup visualToFrozenDrawingGroup(this Visual visual) + { + DrawingGroup dg = new DrawingGroup(); + visual.visualToFrozenDrawingGroup(dg); + return dg; + } #endregion } } \ No newline at end of file diff --git a/ScreenToGif/Windows/Board.xaml.cs b/ScreenToGif/Windows/Board.xaml.cs index 605fcdce..4e1d6e52 100644 --- a/ScreenToGif/Windows/Board.xaml.cs +++ b/ScreenToGif/Windows/Board.xaml.cs @@ -3,7 +3,9 @@ using System.Threading; using System.Threading.Tasks; using System.Windows; +using System.Windows.Controls; using System.Windows.Input; +using System.Windows.Media; using System.Windows.Media.Animation; using System.Windows.Media.Imaging; using ScreenToGif.ImageUtil; @@ -51,7 +53,7 @@ public partial class Board #endregion - #region Inicialization + #region Initialization public Board(bool hideBackButton = true) { @@ -371,6 +373,9 @@ private void AutoFitButtons() private void Normal_Elapsed(object sender, EventArgs e) { var fileName = $"{Project.FullPath}{FrameCount}.png"; + + // We call this only when we do a capture for efficiency's sake. + MainInkCanvas.UpdateInkOverlay(); //TODO: GetRender fails to create useful image when the control has decimals values as size. From 8375b57067933dd1de6fae9c1882d34c5887ece6 Mon Sep 17 00:00:00 2001 From: pntmass Date: Sat, 19 Oct 2019 16:35:53 -0400 Subject: [PATCH 2/8] Removed unused members --- ScreenToGif/Controls/InkCanvasExtended.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/ScreenToGif/Controls/InkCanvasExtended.cs b/ScreenToGif/Controls/InkCanvasExtended.cs index b03badd9..84284238 100644 --- a/ScreenToGif/Controls/InkCanvasExtended.cs +++ b/ScreenToGif/Controls/InkCanvasExtended.cs @@ -45,8 +45,6 @@ public InkCanvasExtended() private Image InkingImage { get; } - private double _prevHeight; - private double _prevWidth; /// /// Updates the inking overlay from the real-time inking thread /// From 160411a87b660efe9ae5e21b5fb68e433348e5f4 Mon Sep 17 00:00:00 2001 From: pntmass Date: Sun, 20 Oct 2019 19:56:57 -0400 Subject: [PATCH 3/8] Fixed reset problem --- ScreenToGif/Windows/Board.xaml.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ScreenToGif/Windows/Board.xaml.cs b/ScreenToGif/Windows/Board.xaml.cs index 4e1d6e52..3fc0fc7a 100644 --- a/ScreenToGif/Windows/Board.xaml.cs +++ b/ScreenToGif/Windows/Board.xaml.cs @@ -403,6 +403,8 @@ private void LightWindow_SizeChanged(object sender, SizeChangedEventArgs e) private void DiscardButton_Click(object sender, RoutedEventArgs e) { + MainInkCanvas.ResetInkOverlay(); + _capture.Stop(); FrameRate.Stop(); FrameCount = 0; From 3fd19c108d3cc055ff602511795de82804cdab40 Mon Sep 17 00:00:00 2001 From: pntmass Date: Sun, 20 Oct 2019 19:58:41 -0400 Subject: [PATCH 4/8] Code cleanup and discard re-init of overlay --- ScreenToGif/Controls/InkCanvasExtended.cs | 38 ++++++++++++++++------- ScreenToGif/ImageUtil/ImageMethods.cs | 10 ------ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/ScreenToGif/Controls/InkCanvasExtended.cs b/ScreenToGif/Controls/InkCanvasExtended.cs index 84284238..6a362ad7 100644 --- a/ScreenToGif/Controls/InkCanvasExtended.cs +++ b/ScreenToGif/Controls/InkCanvasExtended.cs @@ -16,6 +16,9 @@ namespace ScreenToGif.Controls /// public class InkCanvasExtended : InkCanvas { + /// + /// Base Constructor pluse creation of the Inking Overlay + /// public InkCanvasExtended() : base() { @@ -43,8 +46,19 @@ public InkCanvasExtended() public static readonly DependencyProperty EraserShapeProperty = DependencyProperty.Register("EraserShape", typeof (StylusShape), typeof (InkCanvasExtended), new UIPropertyMetadata(new RectangleStylusShape(10, 10), OnEraserShapePropertyChanged)); + /// + /// Overlay Image that receives DrawingGroup contents from the real-time inking thread upon UpdateInkOverlay() + /// private Image InkingImage { get; } + /// + /// Re-initializes InkingImage (the overlay that receives DrawingGroups from the inking thread) + /// + public void ResetInkOverlay() + { + InkingImage.Source = new DrawingImage(new DrawingGroup()); + } + /// /// Updates the inking overlay from the real-time inking thread /// @@ -55,12 +69,12 @@ public void UpdateInkOverlay() // momentarily when adjusting the margins later on. // Also good for when returning when no inking exists, but could do immediately // before the return in that case. - InkingImage.Source = new DrawingImage(new DrawingGroup()); + ResetInkOverlay(); // The following two HostVisuals are children of the _mainContainerVisual in this thread, // and these won't normally get composed into the UI thread from the inking thread's // element tree until a stroke is complete. - var myArgs = new[] + var rawInkHostVisuals = new[] { typeof(DynamicRenderer).GetField("_rawInkHostVisual1", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(this.DynamicRenderer ), typeof(DynamicRenderer).GetField("_rawInkHostVisual2", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(this.DynamicRenderer ) @@ -78,7 +92,7 @@ public void UpdateInkOverlay() // The ThreadDispatcher derives from System.Windows.Threading.Dispatcher, // which is a little simpler than using more reflection to get the derived type. - System.Windows.Threading.Dispatcher lDispatcher = lDynamicDispProperty.GetValue(lRenderingThread, null) as System.Windows.Threading.Dispatcher; + Dispatcher dispatcher = lDynamicDispProperty.GetValue(lRenderingThread, null) as Dispatcher; if (lRenderingThread == null) { return; @@ -87,27 +101,27 @@ public void UpdateInkOverlay() // We invoke the inking thread to get the visual targets from the real time host visuals, then // use BFS to grab all the DrawingGroups as frozen into a new DrawingGroup, which we return // as frozen to the UI thread. - DrawingGroup inkDrawingGroup = lDispatcher.Invoke(DispatcherPriority.Send, - (DispatcherOperationCallback)delegate (object args) + DrawingGroup inkDrawingGroup = dispatcher.Invoke(DispatcherPriority.Send, + (DispatcherOperationCallback)delegate (object rawInkHVs) { // We've got the field references from the other thread now, so we just get their // VisualTarget properties, which is where we'll make a magical RootVisual call. - object[] lObjects = args as object[]; - PropertyInfo lVtProperty = lObjects[0].GetType().GetProperty("VisualTarget", BindingFlags.Instance | + object[] lObjects = rawInkHVs as object[]; + PropertyInfo vtProperty = lObjects[0].GetType().GetProperty("VisualTarget", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); - VisualTarget VisualTarget1 = lVtProperty.GetValue(lObjects[0], null) as VisualTarget; - VisualTarget VisualTarget2 = lVtProperty.GetValue(lObjects[1], null) as VisualTarget; + VisualTarget visualTarget1 = vtProperty.GetValue(lObjects[0], null) as VisualTarget; + VisualTarget visualTarget2 = vtProperty.GetValue(lObjects[1], null) as VisualTarget; // The RootVisual builds the visual when property get is called. We then // all all its descendent DrawingGroups to this DrawingGroup that we are // just using as a collection to return from the thread. DrawingGroup drawingGroups = new DrawingGroup(); - VisualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); - VisualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); + visualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); + visualTarget2?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); return drawingGroups.GetAsFrozen(); }, - myArgs) as DrawingGroup; + rawInkHostVisuals) as DrawingGroup; // This is a little jenky, but we need to set the image margins otherwise diff --git a/ScreenToGif/ImageUtil/ImageMethods.cs b/ScreenToGif/ImageUtil/ImageMethods.cs index 76cc7ab7..6eb1fad6 100644 --- a/ScreenToGif/ImageUtil/ImageMethods.cs +++ b/ScreenToGif/ImageUtil/ImageMethods.cs @@ -1531,16 +1531,6 @@ static public void visualToFrozenDrawingGroup(this Visual visual, DrawingGroup d } } } - - /// - /// Gets a DrawingGroup with frozen copies of all the visual's descendent DrawingGroups - /// - static public DrawingGroup visualToFrozenDrawingGroup(this Visual visual) - { - DrawingGroup dg = new DrawingGroup(); - visual.visualToFrozenDrawingGroup(dg); - return dg; - } #endregion } } \ No newline at end of file From f85b908d4c8d0399a107ff99902a699fc6e24f96 Mon Sep 17 00:00:00 2001 From: pntmass Date: Sat, 26 Oct 2019 09:49:11 -0400 Subject: [PATCH 5/8] Fixed overlay distortion issues by tweaking DrawingGroup instead of margins --- ScreenToGif/Controls/InkCanvasExtended.cs | 35 ++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ScreenToGif/Controls/InkCanvasExtended.cs b/ScreenToGif/Controls/InkCanvasExtended.cs index 6a362ad7..7b112c79 100644 --- a/ScreenToGif/Controls/InkCanvasExtended.cs +++ b/ScreenToGif/Controls/InkCanvasExtended.cs @@ -31,6 +31,7 @@ public InkCanvasExtended() // element and once its stroke is brought over to the UI thread it will // be in front of previous strokes. Canvas.SetZIndex(InkingImage, 99); + } /// /// Gets or set the eraser shape @@ -51,6 +52,16 @@ public InkCanvasExtended() /// private Image InkingImage { get; } + /// + /// Tiny transparent corner point to prevent whitespace before DrawingGroup.TopLeft being cropped by Image control + /// + private readonly GeometryDrawing TransparentCornerPoint = + new GeometryDrawing( + new SolidColorBrush(Color.FromArgb(0, 255, 255, 255)), + new Pen(Brushes.White, 0.1), + new EllipseGeometry(new Point(0, 0), 0.1, 0.1) + ).GetAsFrozen() as GeometryDrawing; + /// /// Re-initializes InkingImage (the overlay that receives DrawingGroups from the inking thread) /// @@ -117,28 +128,20 @@ public void UpdateInkOverlay() // all all its descendent DrawingGroups to this DrawingGroup that we are // just using as a collection to return from the thread. DrawingGroup drawingGroups = new DrawingGroup(); + + // This is a little jenky: + // We add a point in the top left so the DrawingGroup will be + // appropriately offset from the top left of the Image container. + drawingGroups.Children.Add(TransparentCornerPoint); + + // We try to add both the visualTagets (in case both are active) visualTarget1?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); visualTarget2?.RootVisual?.visualToFrozenDrawingGroup(drawingGroups); + return drawingGroups.GetAsFrozen(); }, rawInkHostVisuals) as DrawingGroup; - - // This is a little jenky, but we need to set the image margins otherwise - // the drawing content will be cropped and aligned top-left despite inking - // very far from origin. Because we set image to an empty drawingImage - // earlier, we don't need to worry about things visibly jumping on screen. - // Important note: Negatives are okay. If we set X,Y = (0,0), then the - // DrawingGroup.Bounds.TopLeft = (-3,0) [because user inked off the canvas - // for instance], then the point starting at (-3,0) will be shifted to (0,0) - // and our entire drawing will be moved over by 3 pixels, so here we just - // set the margin to whatever *negative* number the TopLeft has in that case, - // and we care only about going over the actual width/height (e.g. infinity) - var bounds = inkDrawingGroup.Bounds.TopLeft; - var w = System.Math.Min(bounds.X, this.ActualWidth); - var h = System.Math.Min(bounds.Y, this.ActualHeight); - InkingImage.Margin = new System.Windows.Thickness(w, h, 0, 0); - // At this point, just update the image source with the drawing image. InkingImage.Source = new DrawingImage(inkDrawingGroup); } From 6ce3ee81d150ef960b5949676fa9ed000f9d51e7 Mon Sep 17 00:00:00 2001 From: theJosher Date: Sat, 26 Oct 2019 11:49:48 -0400 Subject: [PATCH 6/8] code review - changed order of private and static :smile: --- ScreenToGif/ImageUtil/ImageMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScreenToGif/ImageUtil/ImageMethods.cs b/ScreenToGif/ImageUtil/ImageMethods.cs index 6eb1fad6..766baccc 100644 --- a/ScreenToGif/ImageUtil/ImageMethods.cs +++ b/ScreenToGif/ImageUtil/ImageMethods.cs @@ -1510,7 +1510,7 @@ public static Icon ToIcon(this ImageSource imageSource) /// /// The visual to convert to a DrawingGroup /// The target DrawingGroup to be populated - static public void visualToFrozenDrawingGroup(this Visual visual, DrawingGroup drawingGroup) + public static void visualToFrozenDrawingGroup(this Visual visual, DrawingGroup drawingGroup) { if (visual == null) { From c76a89a618276b651693b8777a3dd4f040d6f79f Mon Sep 17 00:00:00 2001 From: theJosher Date: Sat, 26 Oct 2019 11:55:01 -0400 Subject: [PATCH 7/8] uppercase --- ScreenToGif/ImageUtil/ImageMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScreenToGif/ImageUtil/ImageMethods.cs b/ScreenToGif/ImageUtil/ImageMethods.cs index 766baccc..ce8faf41 100644 --- a/ScreenToGif/ImageUtil/ImageMethods.cs +++ b/ScreenToGif/ImageUtil/ImageMethods.cs @@ -1510,7 +1510,7 @@ public static Icon ToIcon(this ImageSource imageSource) /// /// The visual to convert to a DrawingGroup /// The target DrawingGroup to be populated - public static void visualToFrozenDrawingGroup(this Visual visual, DrawingGroup drawingGroup) + public static void VisualToFrozenDrawingGroup(this Visual visual, DrawingGroup drawingGroup) { if (visual == null) { From cf73ba39173226d475213868be9e03e7ff833003 Mon Sep 17 00:00:00 2001 From: theJosher Date: Sat, 26 Oct 2019 14:00:32 -0400 Subject: [PATCH 8/8] slightly lighter PathGeometry --- ScreenToGif/Controls/InkCanvasExtended.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScreenToGif/Controls/InkCanvasExtended.cs b/ScreenToGif/Controls/InkCanvasExtended.cs index 7b112c79..7699bfa8 100644 --- a/ScreenToGif/Controls/InkCanvasExtended.cs +++ b/ScreenToGif/Controls/InkCanvasExtended.cs @@ -59,7 +59,7 @@ public InkCanvasExtended() new GeometryDrawing( new SolidColorBrush(Color.FromArgb(0, 255, 255, 255)), new Pen(Brushes.White, 0.1), - new EllipseGeometry(new Point(0, 0), 0.1, 0.1) + new LineGeometry(new Point(0, 0), new Point(0.1,0.1)) ).GetAsFrozen() as GeometryDrawing; ///