[log] show invalid utf hex dump in log view too

Author: tstack

src/field_overlay_source.cc

@@ -31,6 +31,7 @@
 
 #include <curl/curl.h>
 
+#include "base/attr_line.builder.hh"
 #include "base/auto_mem.hh"
 #include "base/humanize.time.hh"
 #include "base/injector.hh"
@@ -86,6 +87,33 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
         display = display || this->fos_contexts.top().c_show;
     }
 
+    if (!ll->is_valid_utf()) {
+        auto sub_opts = subline_options{};
+        sub_opts.scrub_invalid_utf8 = false;
+        auto read_res = file->read_line(ll, sub_opts);
+        if (read_res.isOk()) {
+            auto sbr = read_res.unwrap();
+            attr_line_t al;
+            attr_line_builder alb(al);
+
+            alb.append_as_hexdump(sbr.to_string_fragment());
+            this->fos_lines.emplace_back(
+                attr_line_t("  ")
+                    .append(
+                        "Hex dump of line with invalid UTF-8 content"_table_header)
+                    .move());
+            al.split_lines(this->fos_lines);
+            auto first_line = true;
+            for (auto& al_line : this->fos_lines) {
+                if (first_line) {
+                    first_line = false;
+                    continue;
+                }
+                al_line.insert(0, 4, ' ');
+            }
+        }
+    }
+
     if (!display) {
         return;
     }

src/logfile_sub_source.cc

@@ -305,8 +305,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
             sbr.get_metadata().m_has_ansi = false;
         }
     } else {
+        auto sub_opts = subline_options{};
+        sub_opts.scrub_invalid_utf8 = false;
         this->lss_token_value
-            = this->lss_token_file->read_line(this->lss_token_line)
+            = this->lss_token_file->read_line(this->lss_token_line, sub_opts)
                   .map([](auto sbr) { return to_string(sbr); })
                   .unwrapOr({});
         if (this->lss_token_line->has_ansi()) {

test/Makefile.am

@@ -372,6 +372,7 @@ dist_noinst_DATA = \
 	logfile_invalid_json2.json \
 	logfile_mixed_json2.json \
 	logfile_mux_zookeeper.0 \
+	logfile_invalid_utf8.0 \
 	logfile_java.0 \
 	logfile_java.1 \
 	logfile_journald.json \

test/expected/Makefile.am

@@ -494,6 +494,8 @@ dist_noinst_DATA = \
     test_logfile.sh_7c2e11488bccc59458b5775db4b90de964858259.out \
     test_logfile.sh_82d65f4dac07b1623f8202444d5b9144096e8243.err \
     test_logfile.sh_82d65f4dac07b1623f8202444d5b9144096e8243.out \
+    test_logfile.sh_83b6f3269785adf92e60154e959ab14fba7cfb7f.err \
+    test_logfile.sh_83b6f3269785adf92e60154e959ab14fba7cfb7f.out \
     test_logfile.sh_8a5e754cd471e5fdcdaede49c9290903acd7aad6.err \
     test_logfile.sh_8a5e754cd471e5fdcdaede49c9290903acd7aad6.out \
     test_logfile.sh_977cd36c516d568070253202a0ba81f545cb216f.err \

test/expected/test_logfile.sh_83b6f3269785adf92e60154e959ab14fba7cfb7f.err

@@ -0,0 +1,7 @@
+		<node>???????????���??</node>
+▌ Line 3 metadata
+  Hex dump of line with invalid UTF-8 content
+     09 09 3c 6e 6f 64 65 3e  3f 3f 3f 3f 3f 3f 3f 3f  __<node> ????????
+     3f 3f 3f ed b9 a4 3f 3f  3c 2f 6e 6f 64 65 3e     ???×××?? </node>
+	</sample>
+</tiptop>

test/expected/test_logfile.sh_83b6f3269785adf92e60154e959ab14fba7cfb7f.out

@@ -0,0 +1,6 @@
+2015-03-12T23:16:52.071:INFO:com.root:Response :
+<tiptop>
+	<sample>
+		<node>???????????���??</node>
+	</sample>
+</tiptop>

test/logfile_invalid_utf8.0

@@ -4,6 +4,10 @@ export TZ=UTC
 echo ${top_srcdir}
 echo ${top_builddir}
 
+run_cap_test ${lnav_test} -n \
+    -c ':goto 3' \
+    ${test_dir}/logfile_invalid_utf8.0
+
 printf '#Date:\t20\x800-2-02\n0\n' | run_cap_test \
     env TEST_COMMENT="short timestamp" ${lnav_test} -n