diff --git a/tests/test_match.py b/tests/test_match.py index 74ef44b8d5..2a4bd924d3 100644 --- a/tests/test_match.py +++ b/tests/test_match.py @@ -7,6 +7,7 @@ Run from the repository root: python3 -m unittest discover -s tests -v """ +import codecs import errno import io import os @@ -60,6 +61,17 @@ def test_bare_names_and_crlf_and_bom_are_taken(self): path.write_bytes(b"\xef\xbb\xbfcom.example.first\r\n com.example.second \r\n") self.assertEqual(["com.example.first", "com.example.second"], match.read_packages(path)) + def test_utf16_package_files_with_a_bom_are_read(self): + text = ("package:com.example.second\r\n" + "package:/system/app/Foo/base.apk=com.example.first\r\n" + "com.example.second\r\n") + for encoding, bom in (("utf-16-le", codecs.BOM_UTF16_LE), + ("utf-16-be", codecs.BOM_UTF16_BE)): + with self.subTest(encoding=encoding): + path = self.base / "p.txt" + path.write_bytes(bom + text.encode(encoding)) + self.assertEqual(["com.example.first", "com.example.second"], match.read_packages(path)) + def test_lines_that_are_not_package_names_are_skipped(self): path = self.write("p.txt", "error: device offline\n\npackage:com.example.ok\nnot_a_package\n" "rm -rf /\npackage:bad;name.x\n") @@ -286,6 +298,31 @@ def test_runs_isolated_and_exits_0(self): self.assertIn("com.example.ads", result.stdout) self.assertNotIn("Traceback", result.stderr) + def test_package_encodings_produce_the_same_tally(self): + self.add_list("a", "com.example.ads # ads\n# com.example.store # updates\n" + "com.android.packageinstaller\n") + text = ("package:com.example.store\r\npackage:com.example.ads\r\n" + "package:com.android.packageinstaller\r\npackage:com.example.ads\r\n") + expected = None + for encoding, bom in (("utf-8", b""), ("utf-8-sig", b""), + ("utf-16-le", codecs.BOM_UTF16_LE), + ("utf-16-be", codecs.BOM_UTF16_BE)): + with self.subTest(encoding=encoding): + packages = self.base / "p.txt" + original = bom + text.encode(encoding) + packages.write_bytes(original) + result = self.run_script(str(packages), "--examples", str(self.examples)) + self.assertEqual(0, result.returncode, result.stderr) + if expected is None: + expected = result.stdout + self.assertEqual(expected, result.stdout) + rows = {line.split()[0]: line.split()[1:] for line in result.stdout.splitlines()[1:]} + self.assertEqual(["1", "0", "ads"], rows["com.example.ads"]) + self.assertEqual(["0", "1", "updates"], rows["com.example.store"]) + self.assertEqual(["never"], rows["com.android.packageinstaller"]) + self.assertIn("3 packages, counted over 1 lists", result.stderr) + self.assertEqual(original, packages.read_bytes()) + def test_an_empty_examples_path_uses_the_current_directory(self): self.add_list("a", "com.example.ads # ads\n# com.example.store # updates\n") self.add_list("b", "com.example.ads # ads\ncom.example.store # updates\n") diff --git a/tools/match.py b/tools/match.py index b4fba10732..f61c6fa37e 100644 --- a/tools/match.py +++ b/tools/match.py @@ -14,6 +14,7 @@ It reads files only: no TV, no ADB client, nothing written. It ranks nothing and suggests no list; the counts are what the lists say, not a verdict for this TV. +Package files may be UTF-8 (with or without a BOM), or BOM-marked UTF-16 as Windows PowerShell writes. Exit codes: 0 the table was printed, 2 the packages file, the examples folder or a list in it could not be read. @@ -37,9 +38,12 @@ def read_packages(path): name on its own line is taken too, so a hand-trimmed file works. """ try: - text = Path(path).read_text(encoding="utf-8-sig", errors="replace") + data = Path(path).read_bytes() except OSError as e: raise SystemExit(f"could not read {path} ({e.strerror or e}): pass a saved `pm list packages`") + # PowerShell 5.1 redirection writes UTF-16; its BOM identifies the byte order. + encoding = "utf-16" if data.startswith((b"\xff\xfe", b"\xfe\xff")) else "utf-8-sig" + text = data.decode(encoding, errors="replace") names = set() for line in text.splitlines(): line = line.strip()