fix: optimized image download (#40)

* fix: add additional checks for image fetching

* add tags for release

* feat: add image export disable flag in cmd

* add help message and proper documentation

* fix: infinite loop when fetching images from note container

* chore: update packages

Author: nogiszd

README.md

@@ -33,14 +33,17 @@ Download [latest release](https://github.com/nogiszd/xiaomi-note-exporter/releas
 ```
 Options:
   -h, --help
-        Show this help message.
+        Show this help message and exit
 
   -d, --domain <domain> (default: us.i.mi.com)
         Mi Notes domain that you were redirected to.
 
   -s, --split <timestamp> (default: dd-MM-yyyy_HH-mm-ss)
         Split notes into separate files with provided timestamp format. Must be compatible with:
         https://learn.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings
+
+  -di, --disable-images
+        Disable default image export behavior, images will not be downloaded from the notes.
 ```
 
 ## 🗒️ Note splitting
@@ -51,6 +54,12 @@ With `-s` flag you can enable note splitting - which exports notes to separate d
 
 This enables user to input specific format for timestamp - **but it must be compatible with [.NET specification](https://learn.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings)**.
 
+## 🖼️ Image export
+
+Since `v1.7.0` this app exports images present in notes by default.
+
+You can disable this feature by `-di` flag in CLI args.
+
 ---
 
 ## 🔧 How about maintenance?

xiaomiNoteExporter/ConsoleHelp.cs

@@ -12,9 +12,10 @@ internal class ConsoleHelp(Version? version)
         $"{"Xiaomi Note Exporter".Pastel(Color.FromArgb(252, 106, 0))} {version.GetVersionString()}\n",
         $"Usage: xiaomiNoteExporter.exe {"[options]".Pastel(Color.DimGray)}\n",
         "Options:",
-        "  -h, --help\t\tShow this help message and exit\n",
+        "  -h, --help\n\tShow this help message and exit\n",
         $"  -d, --domain <domain> {"(default: us.i.mi.com)".Pastel(Color.DimGray)}\n\tMi Notes domain that you were redirected to.\n",
-        $"  -s, --split <timestamp> {"(default: dd-MM-yyyy_HH-mm-ss)".Pastel(Color.DimGray)}\n\tSplit notes into separate files with provided timestamp format. Must be compatible with:\n\thttps://learn.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings"
+        $"  -s, --split <timestamp> {"(default: dd-MM-yyyy_HH-mm-ss)".Pastel(Color.DimGray)}\n\tSplit notes into separate files with provided timestamp format. Must be compatible with:\n\thttps://learn.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings\n",
+        $"  -di, --disable-images\n\tDisable default image export behavior, images will not be downloaded from the notes.\n",
     };
 
     public void Print()

xiaomiNoteExporter/DriverHelpers.cs

@@ -0,0 +1,73 @@
+using OpenQA.Selenium;
+using OpenQA.Selenium.Support.UI;
+
+namespace xiaomiNoteExporter;
+
+public static class DriverHelpers
+{
+    public static IReadOnlyCollection<IWebElement> TryFindImages(IWebElement scope)
+    {
+        // temporarily set implicit wait to zero
+        var driver = ((IWrapsDriver)scope).WrappedDriver;
+        var timeouts = driver.Manage().Timeouts();
+        var originalImplicitWait = timeouts.ImplicitWait;
+        timeouts.ImplicitWait = TimeSpan.Zero;
+
+        try
+        {
+            var found = scope.FindElements(By.CssSelector(".image-view img"));
+            return found.Count > 0 ? found : Array.Empty<IWebElement>();
+        }
+        finally
+        {
+            // restore original implicit wait timeout
+            timeouts.ImplicitWait = originalImplicitWait;
+        }        
+    }
+
+    public static void WaitUntilImagesAreRealAndLoaded(IWebDriver driver, IReadOnlyCollection<IWebElement> imgs, TimeSpan periodPerItem)
+    {
+        foreach (var img in imgs)
+        {
+            var w = new WebDriverWait(driver, periodPerItem);
+            w.IgnoreExceptionTypes(typeof(StaleElementReferenceException), typeof(NoSuchElementException));
+            w.Until(d => IsRealImageLoaded(d, img));
+        }
+    }
+
+    public static string GetCurrentSrc(IWebDriver driver, IWebElement img)
+    {
+        try
+        {
+            var js = (IJavaScriptExecutor)driver;
+            var src = (string?)js.ExecuteScript("return arguments[0].currentSrc || arguments[0].src || '';", img);
+            return src ?? string.Empty;
+        }
+        catch
+        {
+            return img.GetAttribute("src") ?? string.Empty;
+        }
+    }
+
+    private static bool IsRealImageLoaded(IWebDriver driver, IWebElement imgEl)
+    {
+        try
+        {
+            var js = (IJavaScriptExecutor)driver;
+
+            var src = (string?)js.ExecuteScript("return arguments[0].currentSrc || arguments[0].src || '';", imgEl) ?? "";
+            if (string.IsNullOrWhiteSpace(src)) return false;
+
+            if (src.StartsWith("data:image/svg+xml", StringComparison.OrdinalIgnoreCase)) return false;
+
+            var complete = (bool)(js.ExecuteScript("return arguments[0].complete === true;", imgEl) ?? false);
+            var hasSize = (bool)(js.ExecuteScript("return (arguments[0].naturalWidth||0) > 0;", imgEl) ?? false);
+
+            return complete && hasSize;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+}

xiaomiNoteExporter/Program.cs

@@ -17,6 +17,8 @@ class Program
     static bool _shouldSplit = false;
     static string _timestampFormat = "dd-MM-yyyy_HH-mm-ss";
 
+    static bool _disableImages = false;
+
     readonly static Driver _driver = new(Array.Empty<string>());
     static ChromeDriver? driver;
 
@@ -52,14 +54,14 @@ public static void Main(string[] args)
             $"{"Xiaomi Note Exporter".Pastel(Color.FromArgb(252, 106, 0))} - Export your notes to {"Markdown".Pastel(Color.SkyBlue)}!\n"
             );
 
-        string? domain = _shouldAskForDomain 
+        string? domain = _shouldAskForDomain
             ? new Prompt(
                 $"{"[OPTIONAL]".Pastel(Color.DimGray)} Input Mi Notes domain that you were redirected to (default \"{defaultDomain}\"):",
                 defaultDomain
-                ).Ask() 
+                ).Ask()
             : defaultDomain;
 
-        new Scraper(driver, ShutdownHandler_Handler).Start(domain, _timestampFormat, _shouldSplit);
+        new Scraper(driver, ShutdownHandler_Handler).Start(domain, _timestampFormat, _shouldSplit, !_disableImages);
     }
 
     private static void ShowHelp() => new ConsoleHelp(appVersion).Print();
@@ -72,55 +74,56 @@ private static void ParseArgs(string[] args)
 
             if (arg.Includes("-d", "--domain"))
             {
-                if (i + 1 < args.Length)
+                if (TryGetArgValue(args, i, out var domain) && !string.IsNullOrEmpty(domain))
                 {
-                    string domain = args[i + 1];
-
-                    if (!string.IsNullOrEmpty(domain))
-                    {
-                        defaultDomain = domain; // set global domain to the provided value
-                        _shouldAskForDomain = false; // shouldn't ask for domain, since it was provided as argument
-                    } 
-                    else
-                    {
-                        Console.WriteLine($"{"[ERROR]".Pastel(Color.Red)} Provided domain is invalid.");
-                        Environment.Exit(1);
-                    }
+                    defaultDomain = domain; // set global domain to the provided value
+                    _shouldAskForDomain = false; // shouldn't ask for domain, since it was provided as argument
 
                     i++; // skip next argument - this was a value
-                } 
+                }
                 else
                 {
                     Console.WriteLine($"{"[ERROR]".Pastel(Color.Red)} Domain address is required with domain flag.");
                     Environment.Exit(1);
                 }
-            } 
+            }
             else if (arg.Includes("-s", "--split"))
             {
-                if (i + 1 < args.Length)
+                if (TryGetArgValue(args, i, out var timestampFormat) && !string.IsNullOrEmpty(timestampFormat))
                 {
-                    string timestampFormat = args[i + 1];
-
-                    if (!string.IsNullOrEmpty(timestampFormat))
+                    try
                     {
-                        try
-                        {
-                            DateTime.Now.ToString(timestampFormat);
-                        }
-                        catch (FormatException)
-                        {
-                            Console.WriteLine($"{"[ERROR]".Pastel(Color.Red)} Invalid timestamp format.");
-                            Environment.Exit(1);
-                        }
-
-                        _timestampFormat = timestampFormat;
+                        DateTime.Now.ToString(timestampFormat);
                     }
+                    catch (FormatException)
+                    {
+                        Console.WriteLine($"{"[ERROR]".Pastel(Color.Red)} Invalid timestamp format.");
+                        Environment.Exit(1);
+                    }
+
+                    _timestampFormat = timestampFormat;
 
                     i++; // skip next argument - this was a value
                 }
 
                 _shouldSplit = true; // if flag is present, split is enabled (even if no value is provided)
             }
+            else if (arg.Includes("-di", "--disable-images"))
+            {
+                _disableImages = true;
+            }
+        }
+    }
+
+    private static bool TryGetArgValue(string[] args, int index, out string value)
+    {
+        if (index + 1 < args.Length)
+        {
+            value = args[index + 1];
+            return true;
         }
+
+        value = string.Empty;
+        return false;
     }
 }

xiaomiNoteExporter/Scraper.cs

@@ -30,7 +30,7 @@ public partial class Scraper(ChromeDriver driver, Action shutdownHandler)
     /// <param name="domain">Domain address to be visited by <c>ChromeDriver</c>.</param>
     /// <param name="timeStampFormat">Format of the timestamp for file (or directory) name.</param>
     /// <param name="split">If <c>true</c> then notes will be split as separate files.</param>
-    public void Start(string domain, string timeStampFormat, bool split = false)
+    public void Start(string domain, string timeStampFormat, bool split = false, bool exportImages = true)
     {
         _wait = _driver.GetWait(TimeSpan.FromSeconds(10));
 
@@ -56,10 +56,10 @@ public void Start(string domain, string timeStampFormat, bool split = false)
             Console.ReadKey();
         }
 
-        Scrape(timeStampFormat, domain, split);
+        Scrape(timeStampFormat, domain, split, exportImages);
     }
 
-    private void Scrape(string timeStampFormat, string domain, bool split)
+    private void Scrape(string timeStampFormat, string domain, bool split, bool exportImages)
     {
         if (_wait is null)
         {
@@ -138,28 +138,7 @@ private void Scrape(string timeStampFormat, string domain, bool split)
                     string createdString = element.FindElement(By.XPath(@".//div[2]/div[1]")).Text;
 
                     // creation date (calculated from retrieved text)
-                    DateTime createdDate;
-
-                    if (createdString.ToLower().Contains("now"))
-                    {
-                       createdDate = DateTime.Now; // get current date
-                    }
-                    else if (createdString.ToLower().Contains("yesterday"))
-                    {
-                        createdDate = DateTime.Now.AddDays(-1).Date; // get yesterday's date
-                    }
-                    else if (createdString.EndsWith("ago"))
-                    {
-                        createdDate = RelativeTimeParser.Parse(createdString);
-                    }
-                    else if (SimplifiedDateParser.TryParseMdHm(createdString, out DateTime parsedSimple))
-                    {
-                        createdDate = parsedSimple;
-                    }
-                    else
-                    {
-                        createdDate = DateTime.Parse(createdString, new CultureInfo("en-US"));
-                    }
+                    GetCreatedDate(createdString, out DateTime createdDate);
 
                     try
                     {
@@ -190,23 +169,45 @@ private void Scrape(string timeStampFormat, string domain, bool split)
                         title
                         );
 
-                    var embeddedImages = noteContainer.FindElements(By.XPath(@".//div[contains(@class, 'image-view')]/img"));
+                    if (!exportImages)
+                    {
+                        // skip image export if user chose so
+                        ExecuteScroll(notesList, element);
+                        currentNote++;
+                        continue;
+                    }
+
+                    var initialImgs = DriverHelpers.TryFindImages(noteContainer);
 
-                    if (embeddedImages.Count != 0)
+                    if (initialImgs.Count > 0)
                     {
-                        var cookies = _driver.Manage().Cookies.AllCookies;
+                        DriverHelpers.WaitUntilImagesAreRealAndLoaded(_driver, initialImgs, TimeSpan.FromSeconds(3));
 
-                        // IWebElement because non nullish type is needed (force typing)
-                        foreach (var t in embeddedImages.Select((item, idx) => (idx, (IWebElement)item)))
+                        var embeddedImages = noteContainer.FindElements(By.CssSelector(".image-view img"));
+
+                        if (embeddedImages.Count != 0)
                         {
-                            int idx = t.idx;
-                            IWebElement item = t.Item2;
+                            var cookies = _driver.Manage().Cookies.AllCookies;
+
+                            // IWebElement because non nullish type is needed (force typing)
+                            foreach (var t in embeddedImages.Select((item, idx) => (idx, (IWebElement)item)))
+                            {
+                                int idx = t.idx;
+                                IWebElement item = t.Item2;
+
+                                var imgSrc = DriverHelpers.GetCurrentSrc(_driver, item);
 
-                            var imgSrc = item.GetAttribute("src");
-                            string imgName = $"note_img_{idx}_{createdDate.ToString(timeStampFormat)}.png";
-                            string imgPath = Path.Combine(imgDir, imgName);
+                                if (string.IsNullOrWhiteSpace(imgSrc) || imgSrc.Contains("data:"))
+                                {
+                                    // skip base64 images and empty sources
+                                    continue;
+                                }
 
-                            SaveImage(imgPath, imgSrc, domain, cookies);
+                                string imgName = $"note_img_{idx}_{createdDate.ToString(timeStampFormat)}.png";
+                                string imgPath = Path.Combine(imgDir, imgName);
+
+                                SaveImage(imgPath, imgSrc, cookies);
+                            }
                         }
                     }
 
@@ -254,7 +255,7 @@ private static void SaveToFile(string fileName, string content, string? title =
         sw.WriteLine(content);
     }
 
-    private static void SaveImage(string path, string? src, string domain, IEnumerable<OpenQA.Selenium.Cookie> cookies)
+    private static void SaveImage(string path, string? src, IEnumerable<OpenQA.Selenium.Cookie> cookies)
     {
         if (File.Exists(path))
         {
@@ -266,8 +267,6 @@ private static void SaveImage(string path, string? src, string domain, IEnumerab
             CookieContainer = new CookieContainer()
         };
 
-        var uri = new Uri($"https://{domain}{src}");
-
         foreach (var cookie in cookies)
         {
             handler.CookieContainer.Add(
@@ -282,9 +281,33 @@ private static void SaveImage(string path, string? src, string domain, IEnumerab
             byte[] imageBytes = client.GetByteArrayAsync(src).Result;
             File.WriteAllBytes(path, imageBytes);
         }
-        catch (Exception)
+        catch (Exception e)
+        {
+            Console.WriteLine($"\n{"[ERROR]".Pastel(Color.Red)} Couldn't fetch image.\nError: {e.Message}");
+        }
+    }
+
+    private static void GetCreatedDate(string createdString, out DateTime createdDate)
+    {
+        if (createdString.ToLower().Contains("now"))
+        {
+            createdDate = DateTime.Now; // get current date
+        }
+        else if (createdString.ToLower().Contains("yesterday"))
+        {
+            createdDate = DateTime.Now.AddDays(-1).Date; // get yesterday's date
+        }
+        else if (createdString.EndsWith("ago"))
+        {
+            createdDate = RelativeTimeParser.Parse(createdString);
+        }
+        else if (SimplifiedDateParser.TryParseMdHm(createdString, out DateTime parsedSimple))
+        {
+            createdDate = parsedSimple;
+        }
+        else
         {
-            Console.WriteLine($"{"[ERROR]".Pastel(Color.Red)} Couldn't fetch image.");
+            createdDate = DateTime.Parse(createdString, new CultureInfo("en-US"));
         }
     }