tools: read UTF-16 package files in the offline matcher

tests/test_match.py

@@ -7,6 +7,7 @@
 
 Run from the repository root: python3 -m unittest discover -s tests -v
 """
+import codecs
 import errno
 import io
 import os
@@ -60,6 +61,17 @@ def test_bare_names_and_crlf_and_bom_are_taken(self):
         path.write_bytes(b"\xef\xbb\xbfcom.example.first\r\n  com.example.second  \r\n")
         self.assertEqual(["com.example.first", "com.example.second"], match.read_packages(path))
 
+    def test_utf16_package_files_with_a_bom_are_read(self):
+        text = ("package:com.example.second\r\n"
+                "package:/system/app/Foo/base.apk=com.example.first\r\n"
+                "com.example.second\r\n")
+        for encoding, bom in (("utf-16-le", codecs.BOM_UTF16_LE),
+                              ("utf-16-be", codecs.BOM_UTF16_BE)):
+            with self.subTest(encoding=encoding):
+                path = self.base / "p.txt"
+                path.write_bytes(bom + text.encode(encoding))
+                self.assertEqual(["com.example.first", "com.example.second"], match.read_packages(path))
+
     def test_lines_that_are_not_package_names_are_skipped(self):
         path = self.write("p.txt", "error: device offline\n\npackage:com.example.ok\nnot_a_package\n"
                                    "rm -rf /\npackage:bad;name.x\n")
@@ -286,6 +298,31 @@ def test_runs_isolated_and_exits_0(self):
         self.assertIn("com.example.ads", result.stdout)
         self.assertNotIn("Traceback", result.stderr)
 
+    def test_package_encodings_produce_the_same_tally(self):
+        self.add_list("a", "com.example.ads  # ads\n# com.example.store  # updates\n"
+                           "com.android.packageinstaller\n")
+        text = ("package:com.example.store\r\npackage:com.example.ads\r\n"
+                "package:com.android.packageinstaller\r\npackage:com.example.ads\r\n")
+        expected = None
+        for encoding, bom in (("utf-8", b""), ("utf-8-sig", b""),
+                              ("utf-16-le", codecs.BOM_UTF16_LE),
+                              ("utf-16-be", codecs.BOM_UTF16_BE)):
+            with self.subTest(encoding=encoding):
+                packages = self.base / "p.txt"
+                original = bom + text.encode(encoding)
+                packages.write_bytes(original)
+                result = self.run_script(str(packages), "--examples", str(self.examples))
+                self.assertEqual(0, result.returncode, result.stderr)
+                if expected is None:
+                    expected = result.stdout
+                self.assertEqual(expected, result.stdout)
+                rows = {line.split()[0]: line.split()[1:] for line in result.stdout.splitlines()[1:]}
+                self.assertEqual(["1", "0", "ads"], rows["com.example.ads"])
+                self.assertEqual(["0", "1", "updates"], rows["com.example.store"])
+                self.assertEqual(["never"], rows["com.android.packageinstaller"])
+                self.assertIn("3 packages, counted over 1 lists", result.stderr)
+                self.assertEqual(original, packages.read_bytes())
+
     def test_an_empty_examples_path_uses_the_current_directory(self):
         self.add_list("a", "com.example.ads  # ads\n# com.example.store  # updates\n")
         self.add_list("b", "com.example.ads  # ads\ncom.example.store  # updates\n")

tools/match.py

@@ -14,6 +14,7 @@
 
 It reads files only: no TV, no ADB client, nothing written. It ranks nothing and suggests no list; the
 counts are what the lists say, not a verdict for this TV.
+Package files may be UTF-8 (with or without a BOM), or BOM-marked UTF-16 as Windows PowerShell writes.
 
 Exit codes: 0 the table was printed, 2 the packages file, the examples folder or a list in it could not
 be read.
@@ -37,9 +38,12 @@ def read_packages(path):
     name on its own line is taken too, so a hand-trimmed file works.
     """
     try:
-        text = Path(path).read_text(encoding="utf-8-sig", errors="replace")
+        data = Path(path).read_bytes()
     except OSError as e:
         raise SystemExit(f"could not read {path} ({e.strerror or e}): pass a saved `pm list packages`")
+    # PowerShell 5.1 redirection writes UTF-16; its BOM identifies the byte order.
+    encoding = "utf-16" if data.startswith((b"\xff\xfe", b"\xfe\xff")) else "utf-8-sig"
+    text = data.decode(encoding, errors="replace")
     names = set()
     for line in text.splitlines():
         line = line.strip()