Manifest detection is more lenient regarding content-type response header

Author: JudahGabriel

apps/pwabuilder/Common/Constants.cs

@@ -4,11 +4,12 @@ public static class Constants
 {
     public const string DesktopUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0 PWABuilderHttpAgent";
     public const string PwaBuilderAgentHttpClient = "PWABuilderHttpAgent";
-    public static readonly IReadOnlyCollection<string> ManifestMimeTypes = 
+    public static readonly IReadOnlyCollection<string> ManifestMimeTypes =
     [
         "application/manifest+json",
         "application/json",
-        "text/json"
+        "text/json",
+        "application/octet-stream" // Some servers incorrectly serve manifests with this MIME type, but browsers still consider such manifests valid, so we will too.
     ];
     public static readonly IReadOnlyCollection<string> JavascriptMimeTypes = [
         "application/javascript",

apps/pwabuilder/Services/ManifestDetector.cs

@@ -130,7 +130,7 @@ public ManifestDetector(WebStringCache webStringCache, IPuppeteerService puppete
 
         try
         {
-            var manifestJson = await webStringCache.Get(manifestUrl, Constants.ManifestMimeTypes, cancelToken, 1024 * 1024 * 5);
+            var manifestJson = await webStringCache.Get(manifestUrl, Constants.ManifestMimeTypes, logger, cancelToken, 1024 * 1024 * 5);
             if (string.IsNullOrWhiteSpace(manifestJson))
             {
                 logger.LogWarning("Manifest at {manifestUrl} returned empty content.", manifestUrl);
@@ -187,7 +187,7 @@ public ManifestDetector(WebStringCache webStringCache, IPuppeteerService puppete
     {
         try
         {
-            var htmlString = await webStringCache.Get(appUrl, ["text/html"], cancelToken);
+            var htmlString = await webStringCache.Get(appUrl, ["text/html"], logger, cancelToken);
             return htmlString;
         }
         catch (Exception htmlFetchError)

apps/pwabuilder/Services/ServiceWorkerAnalyzer.cs

@@ -254,7 +254,7 @@ private async Task<List<string>> TryFindAndFetchImportScriptsAsync(string code,
             // Fetch the content of each script
             var contents = new List<string>();
             var scriptFetches = normalizedUrls
-                .Select(url => webCache.Get(url, Constants.JavascriptMimeTypes, cancelToken));
+                .Select(url => webCache.Get(url, Constants.JavascriptMimeTypes, logger, cancelToken));
             try
             {
                 var scriptResults = await Task.WhenAll(scriptFetches);

apps/pwabuilder/Services/ServiceWorkerDetector.cs

@@ -135,7 +135,7 @@ public ServiceWorkerDetector(WebStringCache webStringCache, IPuppeteerService pu
 
         try
         {
-            var serviceWorkerJs = await webStringCache.Get(serviceWorkerUrl, Constants.JavascriptMimeTypes, cancelToken, maxSizeInBytes: 1024 * 1024 * 1);
+            var serviceWorkerJs = await webStringCache.Get(serviceWorkerUrl, Constants.JavascriptMimeTypes, logger, cancelToken, maxSizeInBytes: 1024 * 1024 * 1);
             if (serviceWorkerJs == null)
             {
                 logger.LogWarning("Service worker at {serviceWorkerUrl} returned null content.", serviceWorkerUrl);
@@ -189,7 +189,7 @@ public ServiceWorkerDetector(WebStringCache webStringCache, IPuppeteerService pu
     {
         try
         {
-            var html = await webStringCache.Get(appUrl, ["text/html"], cancelToken);
+            var html = await webStringCache.Get(appUrl, ["text/html"], logger, cancelToken);
             return html;
         }
         catch (Exception htmlFetchError)

apps/pwabuilder/Services/WebStringCache.cs

@@ -35,8 +35,10 @@ public WebStringCache(IDatabase redis, IHttpClientFactory httpClientFactory, ILo
     /// <param name="url">The URL to fetch.</param>
     /// <param name="accepts"> The type of content to accept (e.g., "text/html", "application/json").</param>
     /// <param name="cancelToken">The cancellation token.</param>
+    /// <param name="logger">An optional logger to log the results to.</param>
+    /// <param name="maxSizeInBytes">The maximum size of the resource to fetch
     /// <returns>The resource at the given URL.</returns>
-    public async Task<string?> Get(Uri url, IEnumerable<string> accepts, CancellationToken cancelToken, int maxSizeInBytes = defaultMaxSizeInBytes)
+    public async Task<string?> Get(Uri url, IEnumerable<string> accepts, ILogger? logger, CancellationToken cancelToken, int maxSizeInBytes = defaultMaxSizeInBytes)
     {
         var cacheKey = GetCacheKey(url, accepts);
         var cached = await redis.StringGetAsync(cacheKey);
@@ -46,7 +48,7 @@ public WebStringCache(IDatabase redis, IHttpClientFactory httpClientFactory, ILo
         }
 
         // It's not in the cache. Fetch it and if fetch was successful, put it in the cache.
-        var webString = await TryGetResource(url, accepts, cancelToken);
+        var webString = await TryGetResource(url, accepts, maxSizeInBytes, logger ?? this.logger, cancelToken);
         if (webString != null)
         {
             await redis.StringSetAsync(cacheKey, webString, cacheExpiration);
@@ -55,11 +57,11 @@ public WebStringCache(IDatabase redis, IHttpClientFactory httpClientFactory, ILo
         return webString;
     }
 
-    private async Task<string?> TryGetResource(Uri appUrl, IEnumerable<string> accepts, CancellationToken cancelToken)
+    private async Task<string?> TryGetResource(Uri appUrl, IEnumerable<string> accepts, int maxSizeInBytes, ILogger logger, CancellationToken cancelToken)
     {
         try
         {
-            var webString = await http.GetStringAsync(appUrl, accepts, defaultMaxSizeInBytes, cancelToken);
+            var webString = await http.GetStringAsync(appUrl, accepts, maxSizeInBytes, cancelToken);
             if (webString == null)
             {
                 logger.LogWarning("No response received for {appUrl} with accept {accept}.", appUrl, accepts);