Update other categorization

Author: DariusIII

app/Services/Categorization/Categorizers/MiscCategorizer.php

@@ -100,7 +100,10 @@ public function inspectSignals(ReleaseContext|string $context): array
 
         $signalScore = count($markers);
         $isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
-        $lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12;
+        $lowSignal = $signalScore === 0
+            && $isCoreToken
+            && strlen($coreName) >= 12
+            && ! $this->hasStrongWordStructure($name, $coreName);
 
         return [
             'coreName' => $coreName,
@@ -226,6 +229,10 @@ protected function checkLowSignal(string $name): ?CategorizationResult
     {
         $analysis = $this->inspectSignals($name);
 
+        if ($this->hasStrongWordStructure($name, $analysis['coreName'])) {
+            return null;
+        }
+
         if ($this->isZeroVowelLongToken($analysis['coreName'])) {
             return null;
         }
@@ -246,6 +253,23 @@ protected function checkLowSignal(string $name): ?CategorizationResult
         return null;
     }
 
+    protected function hasStrongWordStructure(string $name, string $coreName): bool
+    {
+        return $this->getMaxConsecutiveLetters($coreName) >= 5
+            && $this->hasNormalVowelRatio($coreName)
+            && $this->countAlphabeticWordTokens($name) >= 2;
+    }
+
+    protected function countAlphabeticWordTokens(string $name): int
+    {
+        $tokens = preg_split('/[.\s_-]+/', $this->stripExtensionsForAnalysis($name)) ?: [];
+
+        return count(array_filter(
+            $tokens,
+            static fn (string $token): bool => preg_match('/[a-z]{3,}/i', $token) === 1
+        ));
+    }
+
     protected function checkArchive(string $name): ?CategorizationResult
     {
         if (preg_match('/\.(zip|rar|7z|tar|gz|bz2|xz|tgz|tbz2|cab|iso|img|dmg|pkg|archive)$/i', $name)) {

app/Services/Categorization/Categorizers/PcCategorizer.php

@@ -148,6 +148,10 @@ protected function checkISO(string $name): ?CategorizationResult
 
     protected function check0day(string $name): ?CategorizationResult
     {
+        if (preg_match('/\.(msix|msixbundle|appx|appxbundle|msi)$/i', $name)) {
+            return $this->matched(Category::PC_0DAY, 0.9, '0day_msix_installer');
+        }
+
         // Explicit 0day indicators
         if (preg_match('/[._ -]exe$|[._ -](utorrent|Virtualbox)[._ -]|\b0DAY\b|incl.+crack| DRM$|>DRM</i', $name)) {
             return $this->matched(Category::PC_0DAY, 0.9, '0day_explicit');

app/Traits/DetectsHashedNames.php

@@ -264,6 +264,23 @@ protected function isZeroVowelLongToken(string $coreName, int $minLength = 20):
         return preg_match('/[aeiou]/i', $letterOnly) !== 1;
     }
 
+    /**
+     * Check whether the letter-only portion of a token has a normal vowel ratio.
+     */
+    protected function hasNormalVowelRatio(string $str, float $min = 0.2): bool
+    {
+        $letters = preg_replace('/[^a-z]/i', '', $str);
+
+        if ($letters === '' || strlen($letters) < 5) {
+            return false;
+        }
+
+        preg_match_all('/[aeiou]/i', $letters, $matches);
+        $vowelCount = count($matches[0]);
+
+        return ($vowelCount / strlen($letters)) >= $min;
+    }
+
     /**
      * Check if a string looks like a random/obfuscated string rather than a real title.
      *
@@ -315,7 +332,7 @@ protected function looksLikeRandomString(string $str): bool
     protected function stripExtensionsForAnalysis(string $name): string
     {
         return preg_replace(
-            '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001)$/i',
+            '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001|msix|msixbundle|appx|appxbundle|apk|xap|ipa|deb|rpm|pkg|dmg|exe|msi)$/i',
             '',
             trim($name)
         );

tests/Unit/CategorizePcGameTest.php

@@ -7,10 +7,23 @@
 use App\Models\Category;
 use App\Services\Categorization\Categorizers\PcCategorizer;
 use App\Services\Categorization\ReleaseContext;
+use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\TestCase;
 
 class CategorizePcGameTest extends TestCase
 {
+    /**
+     * @return array<string, array{0: string}>
+     */
+    public static function installerPackageProvider(): array
+    {
+        return [
+            'msix bundle' => ['Adobe Express Photos.Msixbundle'],
+            'appx bundle' => ['Microsoft.To.Do.appxbundle'],
+            'msi package' => ['Utility Installer.msi'],
+        ];
+    }
+
     private PcCategorizer $categorizer;
 
     protected function setUp(): void
@@ -96,4 +109,21 @@ public function test_console_and_mac_not_misclassified_as_pc_game(): void
             }
         }
     }
+
+    #[DataProvider('installerPackageProvider')]
+    public function test_installer_packages_are_classified_as_pc_0day(string $name): void
+    {
+        $context = new ReleaseContext(
+            releaseName: $name,
+            groupId: 0,
+            groupName: '',
+            poster: ''
+        );
+
+        $result = $this->categorizer->categorize($context);
+
+        $this->assertTrue($result->isSuccessful(), "Expected PC 0day match for installer package: $name");
+        $this->assertSame(Category::PC_0DAY, $result->categoryId, "Expected PC_0DAY for installer package: $name");
+        $this->assertSame('0day_msix_installer', $result->matchedBy);
+    }
 }

tests/Unit/HashedReleaseCategorizationTest.php

@@ -132,6 +132,21 @@ public static function legitimateNamesProvider(): array
             'TV episode' => ['Show.Name.S03E05.720p.HDTV.x264-GROUP', 'alt.binaries.hdtv'],
             'Music album' => ['Artist.Name-Album.Title-2024-FLAC-GROUP', 'alt.binaries.sounds.mp3'],
             'Game release' => ['Starfield-RUNE', 'alt.binaries.games'],
+            'Readable software package' => ['Microsoft Office Suite Installer', 'alt.binaries.warez'],
+            'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx'],
+        ];
+    }
+
+    /**
+     * Readable software-like names that must not be classified as hashed by the misc categorizer.
+     *
+     * @return array<string, array{0: string}>
+     */
+    public static function readableSoftwareNamesProvider(): array
+    {
+        return [
+            'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle'],
+            'Office installer words' => ['Microsoft Office Suite Installer'],
         ];
     }
 
@@ -202,6 +217,18 @@ public function test_misc_categorizer_detects_gibberish_names(string $name, stri
         $this->assertSame(Category::OTHER_HASHED, $result->categoryId, "Expected OTHER_HASHED for: $name");
     }
 
+    #[DataProvider('readableSoftwareNamesProvider')]
+    public function test_misc_categorizer_does_not_hash_readable_software_names(string $name): void
+    {
+        $categorizer = new MiscCategorizer;
+        $context = new ReleaseContext(releaseName: $name, groupId: 0);
+        $result = $categorizer->categorize($context);
+
+        $this->assertFalse($result->isSuccessful(), "Readable software name '$name' should not be matched by misc hash heuristics");
+        $this->assertSame(Category::OTHER_MISC, $result->categoryId);
+        $this->assertSame('no_match', $result->matchedBy);
+    }
+
     // ------------------------------------------------------------------
     // Tests: MiscPipe lock mechanism
     // ------------------------------------------------------------------
@@ -289,6 +316,15 @@ public function test_legitimate_releases_are_not_locked(string $name, string $gr
         );
     }
 
+    public function test_adobe_msix_bundle_reaches_pc_0day_in_full_pipeline(): void
+    {
+        $passable = $this->runPipeline('Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx');
+
+        $this->assertFalse($passable->lockedToMisc);
+        $this->assertSame(Category::PC_0DAY, $passable->bestResult->categoryId);
+        $this->assertSame('0day_msix_installer', $passable->bestResult->matchedBy);
+    }
+
     // ------------------------------------------------------------------
     // Tests: shouldStopProcessing() respects the lock
     // ------------------------------------------------------------------