Skip to content

Commit 7454deb

Browse files
authored
[XABT] Use ZipArchive to build APKs. (#9623)
It would appear we no longer require features not available in `System.IO.Compression.ZipArchive` in order to build `.apk`/`.aab` files. Additionally, it appears that `ZipArchive` is noticeably faster than our current usage of `LibZipSharp`. Switch the `BuildArchive` task to preferring `ZipArchive` over `LibZipSharp` when possible for building APKs. The `LibZipSharp` implementation is maintained as a fallback and in case future new `.apk`/`.aab` requirements necessitate its use. Archive sizes for `.apk` increase 1.3%-2.7% which seems acceptable. `.aab` sizes remain roughly the same, likely because `bundletool` repacks them. ### Implementation Notes - `netstandard2.0` does not expose API for examining a `CRC` or `CompressionMethod` of an existing entry in a Zip file, though both `net472` and `net9.0` have private fields. As such, we use reflection to access these private fields. If the runtime we are using does not have these fields, we will fall back to using `LibZipSharp` as we do now. - Abstract our required Zip API to `IZipArchive` so that we can switch between `System.IO.Compression.ZipFile` and `LibZipSharp` as needed. - Due to a bug on .NET Framework where uncompressed files are stored as `Deflate` with a compression level of `0` instead of being stored as `Store`, if we detect that we need to store uncompressed files we will fall back to `LibZipSharp`. This seems to be an uncommon scenario that is not hit by any of our default flows. - Can force fallback to `LibZipSharp` with `$(_AndroidUseLibZipSharp)`=`true`. ### Performance Measurements of the `BuildArchive` task when using the `android` template for initial and incremental build scenarios. #### Debug - FastDev | Scenario | `main` | This PR | | --------------- | ------- | ------- | | Full | 2.428 s | 339 ms | | NoChanges | not run | not run | | ChangeResource | 34 ms | 19 ms | | AddResource | 23 ms | 17 ms | | ChangeCSharp | not run | not run | | ChangeCSharpJLO | not run | not run | | Archive Size | 5,390,140 bytes | 5,537,596 bytes | #### Debug - EmbedAssembliesInApk | Scenario | `main` | This PR | | --------------- | -------- | ------- | | Full | 34.856 s | 4.313 s | | NoChanges | not run | not run | | ChangeResource | 33.385 s | 4.165 s | | AddResource | 32.206 s | 3.963 s | | ChangeCSharp | 32.060 s | 3.979 s | | ChangeCSharpJLO | 33.161 s | 3.997 s | | Archive Size | 76,653,152 bytes | 77,710,097 bytes | #### Release | Scenario | `main` | This PR | | --------------- | ------- | ------- | | Full | 2.195 s | 387 ms | | NoChanges | not run | not run | | ChangeResource | 134 ms | 73 ms | | AddResource | 685 ms | 182 ms | | ChangeCSharp | 705 ms | 142 ms | | ChangeCSharpJLO | 703 ms | 149 ms | | Archive Size | 6,917,153 bytes | 6,917,319 bytes | CI build that falls back to `LibZipSharp` to ensure it still passes our tests: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=10720142
1 parent f7260a7 commit 7454deb

File tree

6 files changed

+468
-107
lines changed

6 files changed

+468
-107
lines changed

src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs

+86-38
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Collections.Generic;
55
using System.IO;
66
using System.Linq;
7+
using System.Runtime.InteropServices;
78
using Microsoft.Android.Build.Tasks;
89
using Microsoft.Build.Framework;
910
using Xamarin.Tools.Zip;
@@ -32,6 +33,8 @@ public class BuildArchive : AndroidTask
3233

3334
public string? UncompressedFileExtensions { get; set; }
3435

36+
public bool UseLibZipSharp { get; set; }
37+
3538
public string? ZipFlushFilesLimit { get; set; }
3639

3740
public string? ZipFlushSizeLimit { get; set; }
@@ -43,8 +46,10 @@ public class BuildArchive : AndroidTask
4346

4447
public override bool RunTask ()
4548
{
49+
var is_aab = string.Compare (AndroidPackageFormat, "aab", true) == 0;
50+
4651
// Nothing needs to be compressed with app bundles. BundleConfig.json specifies the final compression mode.
47-
if (string.Compare (AndroidPackageFormat, "aab", true) == 0)
52+
if (is_aab)
4853
uncompressedMethod = CompressionMethod.Default;
4954

5055
var refresh = true;
@@ -57,7 +62,7 @@ public override bool RunTask ()
5762
refresh = false;
5863
}
5964

60-
using var apk = new ZipArchiveEx (ApkOutputPath, FileMode.Open);
65+
using var apk = ZipArchiveDotNet.Create (Log, ApkOutputPath, System.IO.Compression.ZipArchiveMode.Update, ShouldFallbackToLibZipSharp ());
6166

6267
// Set up AutoFlush
6368
if (int.TryParse (ZipFlushFilesLimit, out int flushFilesLimit)) {
@@ -73,10 +78,9 @@ public override bool RunTask ()
7378
var existingEntries = new List<string> ();
7479

7580
if (refresh) {
76-
for (var i = 0; i < apk.Archive.EntryCount; i++) {
77-
var entry = apk.Archive.ReadEntry ((ulong) i);
78-
Log.LogDebugMessage ($"Registering item {entry.FullName}");
79-
existingEntries.Add (entry.FullName);
81+
foreach (var entry in apk.GetAllEntryNames ()) {
82+
Log.LogDebugMessage ($"Registering item {entry}");
83+
existingEntries.Add (entry);
8084
}
8185
}
8286

@@ -98,6 +102,11 @@ public override bool RunTask ()
98102
Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`");
99103
}
100104

105+
if (entryName == "AndroidManifest.xml" && is_aab) {
106+
Log.LogDebugMessage ("Renaming AndroidManifest.xml to manifest/AndroidManifest.xml");
107+
entryName = "manifest/AndroidManifest.xml";
108+
}
109+
101110
Log.LogDebugMessage ($"Deregistering item {entryName}");
102111
existingEntries.Remove (entryName);
103112

@@ -106,24 +115,28 @@ public override bool RunTask ()
106115
continue;
107116
}
108117

109-
if (apk.Archive.ContainsEntry (entryName)) {
110-
ZipEntry e = apk.Archive.ReadEntry (entryName);
118+
if (apk.ContainsEntry (entryName)) {
119+
var e = apk.GetEntry (entryName);
111120
// check the CRC values as the ModifiedDate is always 01/01/1980 in the aapt generated file.
112121
if (entry.CRC == e.CRC && entry.CompressedSize == e.CompressedSize) {
113122
Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date.");
114123
continue;
115124
}
125+
126+
// Delete the existing entry so we can replace it with the new one.
127+
apk.DeleteEntry (entryName);
116128
}
117129

118130
var ms = new MemoryStream ();
119131
entry.Extract (ms);
132+
ms.Position = 0;
120133
Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}");
121-
apk.Archive.AddStream (ms, entryName, compressionMethod: entry.CompressionMethod);
134+
apk.AddEntry (ms, entryName, entry.CompressionMethod.ToCompressionLevel ());
122135
}
123136
}
124137
}
125138

126-
apk.FixupWindowsPathSeparators ((a, b) => Log.LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`"));
139+
apk.FixupWindowsPathSeparators (Log);
127140

128141
// Add the files to the apk
129142
foreach (var file in FilesToAddToArchive) {
@@ -135,6 +148,8 @@ public override bool RunTask ()
135148
return !Log.HasLoggedErrors;
136149
}
137150

151+
apk_path = apk_path.Replace ('\\', '/');
152+
138153
// This is a temporary hack for adding files directly from inside a .jar/.aar
139154
// into the APK. Eventually another task should be writing them to disk and just
140155
// passing us a filename like everything else.
@@ -145,7 +160,7 @@ public override bool RunTask ()
145160
// eg: "obj/myjar.jar#myfile.txt"
146161
var jar_file_path = disk_path.Substring (0, disk_path.Length - (jar_entry_name.Length + 1));
147162

148-
if (apk.Archive.Any (ze => ze.FullName == apk_path)) {
163+
if (apk.ContainsEntry (apk_path)) {
149164
Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", jar_entry_name, Path.GetFileName (jar_file_path));
150165
continue;
151166
}
@@ -165,7 +180,7 @@ public override bool RunTask ()
165180
}
166181

167182
Log.LogDebugMessage ($"Adding {jar_entry_name} from {jar_file_path} as the archive file is out of date.");
168-
apk.AddEntryAndFlush (data, apk_path);
183+
apk.AddEntry (data, apk_path);
169184
}
170185

171186
continue;
@@ -180,63 +195,96 @@ public override bool RunTask ()
180195
if (string.Compare (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0)
181196
continue;
182197

183-
Log.LogDebugMessage ($"Removing {entry} as it is not longer required.");
184-
apk.Archive.DeleteEntry (entry);
198+
Log.LogDebugMessage ($"Removing {entry} as it is no longer required.");
199+
apk.DeleteEntry (entry);
185200
}
186201

187-
if (string.Compare (AndroidPackageFormat, "aab", true) == 0)
202+
if (is_aab)
188203
FixupArchive (apk);
189204

190205
return !Log.HasLoggedErrors;
191206
}
192207

193-
bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePath, ITaskItem item, List<string> existingEntries)
208+
// .NET Framework has a bug where it doesn't handle uncompressed files correctly.
209+
// It writes them as "compressed" (DEFLATE) but with a compression level of 0. This causes
210+
// issues with Android, which expect uncompressed files to be stored correctly.
211+
// We can work around this by using LibZipSharp, which doesn't have this bug.
212+
// This is only necessary if we're on .NET Framework (MSBuild in VSWin) and we have uncompressed files.
213+
bool ShouldFallbackToLibZipSharp ()
194214
{
195-
var compressionMethod = GetCompressionMethod (item);
196-
existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/'));
215+
// Explicitly requested via MSBuild property.
216+
if (UseLibZipSharp) {
217+
Log.LogDebugMessage ("Falling back to LibZipSharp because '$(_AndroidUseLibZipSharp)' is 'true'.");
218+
return true;
219+
}
197220

198-
if (apk.SkipExistingFile (file, inArchivePath, compressionMethod)) {
199-
Log.LogDebugMessage ($"Skipping {file} as the archive file is up to date.");
221+
// .NET 6+ handles uncompressed files correctly, so we don't need to fallback.
222+
if (RuntimeInformation.FrameworkDescription == ".NET") {
223+
Log.LogDebugMessage ("Using System.IO.Compression because we're running on .NET 6+.");
200224
return false;
201225
}
202226

203-
Log.LogDebugMessage ($"Adding {file} as the archive file is out of date.");
204-
apk.AddFileAndFlush (file, inArchivePath, compressionMethod);
227+
// Nothing is going to get written uncompressed, so we don't need to fallback.
228+
if (uncompressedMethod != CompressionMethod.Store) {
229+
Log.LogDebugMessage ("Using System.IO.Compression because uncompressedMethod isn't 'Store'.");
230+
return false;
231+
}
205232

206-
return true;
233+
// No uncompressed file extensions were specified, so we don't need to fallback.
234+
if (UncompressedFileExtensionsSet.Count == 0) {
235+
Log.LogDebugMessage ("Using System.IO.Compression because no uncompressed file extensions were specified.");
236+
return false;
237+
}
238+
239+
// See if any of the files to be added need to be uncompressed.
240+
foreach (var file in FilesToAddToArchive) {
241+
var file_path = file.ItemSpec;
242+
243+
// Handle files from inside a .jar/.aar
244+
if (file.GetMetadataOrDefault ("JavaArchiveEntry", (string?)null) is string jar_entry_name)
245+
file_path = jar_entry_name;
246+
247+
if (UncompressedFileExtensionsSet.Contains (Path.GetExtension (file_path))) {
248+
Log.LogDebugMessage ($"Falling back to LibZipSharp because '{file_path}' needs to be stored uncompressed.");
249+
return true;
250+
}
251+
}
252+
253+
Log.LogDebugMessage ("Using System.IO.Compression because no files need to be stored uncompressed.");
254+
return false;
255+
}
256+
257+
bool AddFileToArchiveIfNewer (IZipArchive apk, string file, string inArchivePath, ITaskItem item, List<string> existingEntries)
258+
{
259+
var compressionMethod = GetCompressionLevel (item);
260+
existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/'));
261+
262+
return apk.AddFileIfChanged (Log, file, inArchivePath, compressionMethod);
207263
}
208264

209265
/// <summary>
210266
/// aapt2 is putting AndroidManifest.xml in the root of the archive instead of at manifest/AndroidManifest.xml that bundletool expects.
211267
/// I see no way to change this behavior, so we can move the file for now:
212268
/// https://github.com/aosp-mirror/platform_frameworks_base/blob/e80b45506501815061b079dcb10bf87443bd385d/tools/aapt2/LoadedApk.h#L34
213269
/// </summary>
214-
void FixupArchive (ZipArchiveEx zip)
270+
void FixupArchive (IZipArchive zip)
215271
{
216-
if (!zip.Archive.ContainsEntry ("AndroidManifest.xml")) {
272+
if (!zip.ContainsEntry ("AndroidManifest.xml")) {
217273
Log.LogDebugMessage ($"No AndroidManifest.xml. Skipping Fixup");
218274
return;
219275
}
220276

221-
var entry = zip.Archive.ReadEntry ("AndroidManifest.xml");
222277
Log.LogDebugMessage ($"Fixing up AndroidManifest.xml to be manifest/AndroidManifest.xml.");
223278

224-
if (zip.Archive.ContainsEntry ("manifest/AndroidManifest.xml"))
225-
zip.Archive.DeleteEntry (zip.Archive.ReadEntry ("manifest/AndroidManifest.xml"));
279+
if (zip.ContainsEntry ("manifest/AndroidManifest.xml"))
280+
zip.DeleteEntry ("manifest/AndroidManifest.xml");
226281

227-
entry.Rename ("manifest/AndroidManifest.xml");
282+
zip.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml");
228283
}
229284

230-
CompressionMethod GetCompressionMethod (ITaskItem item)
285+
System.IO.Compression.CompressionLevel GetCompressionLevel (ITaskItem item)
231286
{
232-
var compression = item.GetMetadataOrDefault ("Compression", "");
233-
234-
if (compression.HasValue ()) {
235-
if (Enum.TryParse (compression, out CompressionMethod result))
236-
return result;
237-
}
238-
239-
return UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default;
287+
return (UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default).ToCompressionLevel ();
240288
}
241289

242290
HashSet<string> ParseUncompressedFileExtensions ()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
using System;
2+
using System.IO;
3+
using System.IO.Compression;
4+
using Xamarin.Tools.Zip;
5+
6+
namespace Xamarin.Android.Tasks;
7+
8+
static class UtilityExtensions
9+
{
10+
public static System.IO.Compression.CompressionLevel ToCompressionLevel (this CompressionMethod method)
11+
{
12+
switch (method) {
13+
case CompressionMethod.Store:
14+
return System.IO.Compression.CompressionLevel.NoCompression;
15+
case CompressionMethod.Default:
16+
case CompressionMethod.Deflate:
17+
return System.IO.Compression.CompressionLevel.Optimal;
18+
default:
19+
throw new ArgumentOutOfRangeException (nameof (method), method, null);
20+
}
21+
}
22+
23+
public static CompressionMethod ToCompressionMethod (this System.IO.Compression.CompressionLevel level)
24+
{
25+
switch (level) {
26+
case System.IO.Compression.CompressionLevel.NoCompression:
27+
return CompressionMethod.Store;
28+
case System.IO.Compression.CompressionLevel.Optimal:
29+
return CompressionMethod.Deflate;
30+
default:
31+
throw new ArgumentOutOfRangeException (nameof (level), level, null);
32+
}
33+
}
34+
35+
public static FileMode ToFileMode (this ZipArchiveMode mode)
36+
{
37+
switch (mode) {
38+
case ZipArchiveMode.Create:
39+
return FileMode.Create;
40+
case ZipArchiveMode.Update:
41+
return FileMode.Open;
42+
default:
43+
throw new ArgumentOutOfRangeException (nameof (mode), mode, null);
44+
}
45+
}
46+
}

0 commit comments

Comments
 (0)