Improve txt dumper (#70)

mkalkbrenner · web-flow · commit 944465123eb3 · 2025-04-16T07:18:05.000-04:00
diff --git a/include/DMDUtil/Config.h b/include/DMDUtil/Config.h
@@ -51,6 +51,13 @@ class DMDUTILAPI Config
   void SetMaximumUnknownFramesToSkip(int framesToSkip) { m_framesToSkip = framesToSkip; }
   int GetIgnoreUnknownFramesTimeout() { return m_framesTimeout; }
   int GetMaximumUnknownFramesToSkip() { return m_framesToSkip; }
+  bool IsDumpNotColorizedFrames() const { return m_dumpNotColorizedFrames; }
+  void SetDumpNotColorizedFrames(bool dumpNotColorizedFrames) { m_dumpNotColorizedFrames = dumpNotColorizedFrames; }
+  bool IsFilterTransitionalFrames() const { return m_filterTransitionalFrames; }
+  void SetFilterTransitionalFrames(bool filterTransitionalFrames)
+  {
+    m_filterTransitionalFrames = filterTransitionalFrames;
+  }
   bool IsZeDMD() const { return m_zedmd; }
   void SetZeDMD(bool zedmd) { m_zedmd = zedmd; }
   const char* GetZeDMDDevice() const { return m_zedmdDevice.c_str(); }
@@ -96,6 +103,8 @@ class DMDUTILAPI Config
   bool m_pupExactColorMatch;
   int m_framesTimeout;
   int m_framesToSkip;
+  bool m_dumpNotColorizedFrames;
+  bool m_filterTransitionalFrames;
   bool m_zedmd;
   std::string m_zedmdDevice;
   bool m_zedmdDebug;
diff --git a/include/DMDUtil/DMD.h b/include/DMDUtil/DMD.h
@@ -79,16 +79,17 @@ class DMDUTILAPI DMD
 
   enum class Mode
   {
-    Unknown,        // int 0
-    Data,           // int 1
-    RGB24,          // int 2, RGB888
-    RGB16,          // int 3, RGB565
-    AlphaNumeric,   // int 4
-    SerumV1,        // int 5
-    SerumV2_32,     // int 6
-    SerumV2_32_64,  // int 7
-    SerumV2_64,     // int 8
-    SerumV2_64_32,  // int 9
+    Unknown = 0,
+    Data = 1,
+    RGB24 = 2,  // RGB888
+    RGB16 = 3,  // RGB565
+    AlphaNumeric = 4,
+    SerumV1 = 5,
+    SerumV2_32 = 6,
+    SerumV2_32_64 = 7,
+    SerumV2_64 = 8,
+    SerumV2_64_32 = 9,
+    NotColorized = 10,
   };
 
   bool IsSerumMode(Mode mode)
diff --git a/platforms/android/arm64-v8a/external.sh b/platforms/android/arm64-v8a/external.sh
@@ -40,6 +40,7 @@ cmake \
    -B build
 cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/runtime-libs/android/arm64-v8a/libsockpp.so ../../third-party/runtime-libs/android/arm64-v8a/
diff --git a/platforms/ios-simulator/arm64/external.sh b/platforms/ios-simulator/arm64/external.sh
@@ -34,6 +34,7 @@ cmake \
    -B build
 cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp -a third-party/build-libs/ios-simulator/arm64/libsockpp.a ../../third-party/build-libs/ios-simulator/arm64/
diff --git a/platforms/ios/arm64/external.sh b/platforms/ios/arm64/external.sh
@@ -34,6 +34,7 @@ cmake \
    -B build
 cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp -a third-party/build-libs/ios/arm64/libsockpp.a ../../third-party/build-libs/ios/arm64/
diff --git a/platforms/linux/aarch64/external.sh b/platforms/linux/aarch64/external.sh
@@ -36,6 +36,7 @@ cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/runtime-libs/linux/aarch64/libcargs.so ../../third-party/runtime-libs/linux/aarch64/
diff --git a/platforms/linux/x64/external.sh b/platforms/linux/x64/external.sh
@@ -36,6 +36,7 @@ cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/runtime-libs/linux/x64/libcargs.so ../../third-party/runtime-libs/linux/x64/
diff --git a/platforms/macos/arm64/external.sh b/platforms/macos/arm64/external.sh
@@ -36,6 +36,7 @@ cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/runtime-libs/macos/arm64/libcargs.dylib ../../third-party/runtime-libs/macos/arm64/
diff --git a/platforms/macos/x64/external.sh b/platforms/macos/x64/external.sh
@@ -36,6 +36,7 @@ cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/runtime-libs/macos/x64/libcargs.dylib ../../third-party/runtime-libs/macos/x64/
diff --git a/platforms/tvos/arm64/external.sh b/platforms/tvos/arm64/external.sh
@@ -34,6 +34,7 @@ cmake \
    -B build
 cmake --build build -- -j${NUM_PROCS}
 cp src/ZeDMD.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp -a third-party/build-libs/tvos/arm64/libsockpp.a ../../third-party/build-libs/tvos/arm64/
diff --git a/platforms/win/x64/external.sh b/platforms/win/x64/external.sh
@@ -33,6 +33,7 @@ cmake \
 cmake --build build --config ${BUILD_TYPE}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
diff --git a/platforms/win/x86/external.sh b/platforms/win/x86/external.sh
@@ -35,6 +35,7 @@ cmake --build build --config ${BUILD_TYPE}
 cp src/ZeDMD.h ../../third-party/include/
 cp third-party/include/libserialport.h ../../third-party/include/
 cp third-party/include/cargs.h ../../third-party/include/
+cp -r third-party/include/komihash ../../third-party/include/
 cp -r third-party/include/sockpp ../../third-party/include/
 cp third-party/include/FrameUtil.h ../../third-party/include/
 cp third-party/build-libs/win/x86/cargs.lib ../../third-party/build-libs/win/x86/
diff --git a/src/Config.cpp b/src/Config.cpp
@@ -23,6 +23,8 @@ Config::Config()
   m_pupExactColorMatch = true;
   m_framesTimeout = 0;
   m_framesToSkip = 0;
+  m_dumpNotColorizedFrames = false;
+  m_filterTransitionalFrames = false;
   m_zedmd = true;
   m_zedmdDevice.clear();
   m_zedmdDebug = false;
diff --git a/src/DMD.cpp b/src/DMD.cpp
@@ -14,11 +14,13 @@
 #include <algorithm>
 #include <chrono>
 #include <cstring>
+#include <unordered_set>
 
 #include "AlphaNumeric.h"
 #include "FrameUtil.h"
 #include "Logger.h"
 #include "ZeDMD.h"
+#include "komihash/komihash.h"
 #include "pupdmd.h"
 #include "serum-decode.h"
 #include "serum.h"
@@ -236,9 +238,21 @@ void DMD::SetAltColorPath(const char* path) { strcpy(m_altColorPath, path ? path
 
 void DMD::SetPUPVideosPath(const char* path) { strcpy(m_pupVideosPath, path ? path : ""); }
 
-void DMD::DumpDMDTxt() { m_pDumpDMDTxtThread = new std::thread(&DMD::DumpDMDTxtThread, this); }
+void DMD::DumpDMDTxt()
+{
+  if (!m_pDumpDMDTxtThread)
+  {
+    m_pDumpDMDTxtThread = new std::thread(&DMD::DumpDMDTxtThread, this);
+  }
+}
 
-void DMD::DumpDMDRaw() { m_pDumpDMDRawThread = new std::thread(&DMD::DumpDMDRawThread, this); }
+void DMD::DumpDMDRaw()
+{
+  if (m_pDumpDMDRawThread)
+  {
+    m_pDumpDMDRawThread = new std::thread(&DMD::DumpDMDRawThread, this);
+  }
+}
 
 LevelDMD* DMD::CreateLevelDMD(uint16_t width, uint16_t height, bool sam)
 {
@@ -767,6 +781,9 @@ void DMD::SerumThread()
     m_dmdFrameReady.load(std::memory_order_acquire);
     m_stopFlag.load(std::memory_order_acquire);
 
+    Config* const pConfig = Config::GetInstance();
+    bool dumpNotColorizedFrames = pConfig->IsDumpNotColorizedFrames();
+
     while (true)
     {
       std::shared_lock<std::shared_mutex> sl(m_dmdSharedMutex);
@@ -850,6 +867,24 @@ void DMD::SerumThread()
                 prevTriggerId = m_pSerum->triggerID;
               }
             }
+            else if (dumpNotColorizedFrames)
+            {
+              Log(DMDUtil_LogLevel_DEBUG, "Serum: unidentified frame detected");
+
+              auto noSerumUpdate = std::make_shared<Update>();
+              noSerumUpdate->mode = Mode::NotColorized;
+              noSerumUpdate->depth = m_pUpdateBufferQueue[bufferPosition]->depth;
+              noSerumUpdate->width = m_pUpdateBufferQueue[bufferPosition]->width;
+              noSerumUpdate->height = m_pUpdateBufferQueue[bufferPosition]->height;
+              noSerumUpdate->hasData = true;
+              noSerumUpdate->hasSegData = false;
+              noSerumUpdate->hasSegData2 = false;
+              memcpy(
+                  noSerumUpdate->data, m_pUpdateBufferQueue[bufferPosition]->data,
+                  (size_t)m_pUpdateBufferQueue[bufferPosition]->width * m_pUpdateBufferQueue[bufferPosition]->height);
+
+              QueueUpdate(noSerumUpdate, false);
+            }
           }
         }
       }
@@ -1423,10 +1458,15 @@ void DMD::DumpDMDTxtThread()
   uint32_t passed[3] = {0};
   std::chrono::steady_clock::time_point start;
   FILE* f = nullptr;
+  std::unordered_set<uint64_t> seenHashes;
 
   m_dmdFrameReady.load(std::memory_order_acquire);
   m_stopFlag.load(std::memory_order_acquire);
 
+  Config* const pConfig = Config::GetInstance();
+  bool dumpNotColorizedFrames = pConfig->IsDumpNotColorizedFrames();
+  bool filterTransitionalFrames = pConfig->IsFilterTransitionalFrames();
+
   while (true)
   {
     std::shared_lock<std::shared_mutex> sl(m_dmdSharedMutex);
@@ -1450,8 +1490,9 @@ void DMD::DumpDMDTxtThread()
       // Don't use GetNextBufferPosition() here, we need all frames!
       if (++bufferPosition >= DMDUTIL_FRAME_BUFFER_SIZE) bufferPosition = 0;
 
-      if (m_pUpdateBufferQueue[bufferPosition]->depth <= 4 &&
-          m_pUpdateBufferQueue[bufferPosition]->mode == Mode::Data && m_pUpdateBufferQueue[bufferPosition]->hasData)
+      if (m_pUpdateBufferQueue[bufferPosition]->depth <= 4 && m_pUpdateBufferQueue[bufferPosition]->hasData &&
+          ((m_pUpdateBufferQueue[bufferPosition]->mode == Mode::Data && !dumpNotColorizedFrames) ||
+           (m_pUpdateBufferQueue[bufferPosition]->mode == Mode::NotColorized && dumpNotColorizedFrames)))
       {
         bool update = false;
         if (strcmp(m_romName, name) != 0)
@@ -1486,7 +1527,7 @@ void DMD::DumpDMDTxtThread()
                                        .count());
             memcpy(renderBuffer[2], m_pUpdateBufferQueue[bufferPosition]->data, length);
 
-            if (m_pUpdateBufferQueue[bufferPosition]->depth == 2 &&
+            if (filterTransitionalFrames && m_pUpdateBufferQueue[bufferPosition]->depth == 2 &&
                 (passed[2] - passed[1]) < DMDUTIL_MAX_TRANSITIONAL_FRAME_DURATION)
             {
               int i = 0;
@@ -1498,6 +1539,8 @@ void DMD::DumpDMDTxtThread()
               }
               if (i == length)
               {
+                Log(DMDUtil_LogLevel_DEBUG, "DumpDMDTxt: skip transitional frame");
+
                 // renderBuffer[1] is a transitional frame, delete it.
                 memcpy(renderBuffer[1], renderBuffer[2], length);
                 passed[1] += passed[2];
@@ -1509,16 +1552,35 @@ void DMD::DumpDMDTxtThread()
             {
               if (passed[0] > 0)
               {
-                fprintf(f, "0x%08x\r\n", passed[0]);
-                for (int y = 0; y < m_pUpdateBufferQueue[bufferPosition]->height; y++)
+                bool dump = true;
+
+                if (dumpNotColorizedFrames)
+                {
+                  uint64_t hash = komihash(renderBuffer[0], length, 0);
+                  if (seenHashes.find(hash) == seenHashes.end())
+                  {
+                    seenHashes.insert(hash);
+                  }
+                  else
+                  {
+                    Log(DMDUtil_LogLevel_DEBUG, "DumpDMDTxt: skip duplicate frame");
+                    dump = false;
+                  }
+                }
+
+                if (dump)
                 {
-                  for (int x = 0; x < m_pUpdateBufferQueue[bufferPosition]->width; x++)
+                  fprintf(f, "0x%08x\r\n", passed[0]);
+                  for (int y = 0; y < m_pUpdateBufferQueue[bufferPosition]->height; y++)
                   {
-                    fprintf(f, "%x", renderBuffer[0][y * m_pUpdateBufferQueue[bufferPosition]->width + x]);
+                    for (int x = 0; x < m_pUpdateBufferQueue[bufferPosition]->width; x++)
+                    {
+                      fprintf(f, "%x", renderBuffer[0][y * m_pUpdateBufferQueue[bufferPosition]->width + x]);
+                    }
+                    fprintf(f, "\r\n");
                   }
                   fprintf(f, "\r\n");
                 }
-                fprintf(f, "\r\n");
               }
             }
             memcpy(renderBuffer[0], renderBuffer[1], length);
