[applevz] ASIF image support (#4676)

Add support for ASIF images on macOS 26 and later. Conversion from QCOW2
and existing VMs using RAW images are also handled.

As the `qemu-img` binary is required by more than just the `qemu`
backend, it has been factored out of the backend code into the `utils`
directory. It can now be used by the `applevz` and `hyperv` backends
without having the `qemu` backend be compiled.

---

MULTI-2270
MULTI-2273

Author: sharder996

include/multipass/json_utils.h

@@ -167,7 +167,9 @@ inline void tag_invoke(const boost::json::value_from_tag&,
 
 inline QString tag_invoke(const boost::json::value_to_tag<QString>&, const boost::json::value& json)
 {
-    return QString::fromStdString(value_to<std::string>(json));
+    if (json.is_string())
+        return QString::fromStdString(value_to<std::string>(json));
+    return QString::fromStdString(boost::json::serialize(json));
 }
 
 void tag_invoke(const boost::json::value_from_tag&,

…/platform/backends/qemu/qemu_img_utils.h include/multipass/utils/qemu_img_utils.hsrc/platform/backends/qemu/qemu_img_utils.h renamed to include/multipass/utils/qemu_img_utils.h src/platform/backends/qemu/qemu_img_utils.h renamed to include/multipass/utils/qemu_img_utils.h

@@ -40,10 +40,11 @@ class QemuImgException : public std::runtime_error
 Process::UPtr checked_exec_qemu_img(std::unique_ptr<QemuImgProcessSpec> spec,
                                     const std::string& custom_error_prefix = "Internal error",
                                     std::optional<int> timeout = std::nullopt);
+QString get_image_info(const std::filesystem::path& image_path, const QString& key);
 void resize_instance_image(const MemorySize& disk_space, const std::filesystem::path& image_path);
-std::filesystem::path convert_to_qcow_if_necessary(const std::filesystem::path& image_path);
+std::filesystem::path convert(const std::filesystem::path& image_path,
+                              const std::string& target_format);
 void amend_to_qcow2_v3(const std::filesystem::path& image_path);
-std::filesystem::path convert_to_raw(const std::filesystem::path& image_path);
 bool instance_image_has_snapshot(const std::filesystem::path& image_path, QString snapshot_tag);
 QByteArray snapshot_list_output(const std::filesystem::path& image_path);
 void delete_snapshot_from_image(const std::filesystem::path& image_path,

src/daemon/default_vm_image_vault.cpp

@@ -31,6 +31,7 @@
 #include <multipass/rpc/multipass.grpc.pb.h>
 #include <multipass/url_downloader.h>
 #include <multipass/utils.h>
+#include <multipass/utils/qemu_img_utils.h>
 #include <multipass/vm_image.h>
 
 #include <QUrl>
@@ -85,37 +86,7 @@ void delete_image_dir(const mp::Path& image_path)
 
 mp::MemorySize get_image_size(const std::filesystem::path& image_path)
 {
-    QStringList qemuimg_parameters{{"info", MP_PLATFORM.path_to_qstr(image_path)}};
-    auto qemuimg_process = mp::platform::make_process(
-        std::make_unique<mp::QemuImgProcessSpec>(qemuimg_parameters, image_path));
-    auto process_state = qemuimg_process->execute();
-
-    if (!process_state.completed_successfully())
-    {
-        throw std::runtime_error(
-            fmt::format("Cannot get image info: qemu-img failed ({}) with output:\n{}",
-                        process_state.failure_message(),
-                        qemuimg_process->read_all_standard_error()));
-    }
-
-    const auto img_info = QString{qemuimg_process->read_all_standard_output()};
-    const auto pattern = QStringLiteral("^virtual size: .+ \\((?<size>\\d+) bytes\\)\r?$");
-    const auto re = QRegularExpression{pattern, QRegularExpression::MultilineOption};
-
-    mp::MemorySize image_size{};
-
-    const auto match = re.match(img_info);
-
-    if (match.hasMatch())
-    {
-        image_size = mp::MemorySize(match.captured("size").toStdString());
-    }
-    else
-    {
-        throw std::runtime_error{"Could not obtain image's virtual size"};
-    }
-
-    return image_size;
+    return mp::MemorySize(mp::backend::get_image_info(image_path, "virtual-size").toStdString());
 }
 
 void persist_records(const std::unordered_map<std::string, mp::VaultRecord>& records,

src/platform/backends/applevz/CMakeLists.txt

@@ -20,9 +20,13 @@ add_library(applevz_backend STATIC
   applevz_virtual_machine_factory.cpp
   applevz_bridge.mm
   applevz_wrapper.cpp
+  applevz_utils.mm
 )
 
-set_source_files_properties(applevz_bridge.mm PROPERTIES
+set_source_files_properties(
+  applevz_bridge.mm
+  applevz_utils.mm
+  PROPERTIES
   LANGUAGE OBJCXX
   COMPILE_FLAGS "-fobjc-arc"
 )

src/platform/backends/applevz/applevz_utils.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <multipass/memory_size.h>
+#include <multipass/singleton.h>
+
+#include <filesystem>
+
+#define MP_APPLEVZ_UTILS multipass::applevz::AppleVZUtils::instance()
+
+namespace multipass::applevz
+{
+class AppleVZUtils : public Singleton<AppleVZUtils>
+{
+public:
+    using Singleton<AppleVZUtils>::Singleton;
+
+    virtual std::filesystem::path convert_to_supported_format(
+        const std::filesystem::path& image_path,
+        bool destructive = true) const;
+    virtual void resize_image(const MemorySize& disk_space,
+                              const std::filesystem::path& image_path) const;
+    virtual bool macos_at_least(int major, int minor, int patch = 0) const;
+};
+} // namespace multipass::applevz

src/platform/backends/applevz/applevz_utils.mm

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * This file provides some support for creating and resizing ASIF disk images.
+ * There is no official public documentation for the ASIF format. The technical
+ * details used here are based on community reverse engineering. For reference,
+ * see: https://github.com/huven/asif-format
+ */
+
+#include "applevz_utils.h"
+
+#include <applevz/applevz_wrapper.h>
+#include <multipass/file_ops.h>
+#include <multipass/format.h>
+#include <multipass/logging/log.h>
+#include <multipass/utils/qemu_img_utils.h>
+#include <shared/macos/process_factory.h>
+
+#include <Foundation/Foundation.h>
+
+#include <scope_guard.hpp>
+
+#include <algorithm>
+#include <array>
+#include <filesystem>
+#include <fstream>
+#include <stdexcept>
+#include <vector>
+
+namespace mp = multipass;
+namespace mpl = multipass::logging;
+
+namespace
+{
+constexpr auto category = "applevz-utils";
+
+std::string run_process(const QString& program, const QStringList& args, const std::string& desc)
+{
+    mpl::info(category, "Trying to {}", desc);
+
+    auto process = MP_PROCFACTORY.create_process(program, args);
+
+    if (const auto exit_state = process->execute(); !exit_state.completed_successfully())
+        throw std::runtime_error(fmt::format("Failed to {}: {}; Output: {}",
+                                             desc,
+                                             exit_state.failure_message(),
+                                             process->read_all_standard_error()));
+
+    return process->read_all_standard_output().trimmed().toStdString();
+}
+
+bool is_asif_image(const std::filesystem::path& image_path)
+{
+    // ASIF format uses "shdw" magic bytes (0x73686477)
+    const auto file = MP_FILEOPS.open_read(image_path, std::ios::binary);
+    file->exceptions(std::ios::badbit);
+    if (file->fail())
+        throw std::runtime_error(fmt::format("Failed to open file for reading: {}", image_path));
+
+    std::array<char, 4> magic{};
+    file->read(magic.data(), magic.size());
+
+    return file->gcount() == 4 && std::equal(magic.begin(), magic.end(), "shdw");
+}
+
+void resize_asif_image(const std::filesystem::path& image_path, const mp::MemorySize& disk_space)
+{
+    run_process(
+        QStringLiteral("diskutil"),
+        QStringList() << "image" << "resize" << "--size" << QString::number(disk_space.in_bytes())
+                      << MP_PLATFORM.path_to_qstr(image_path),
+        fmt::format("resize ASIF image: {}, size: {}", image_path, disk_space.human_readable()));
+}
+
+void create_asif_from(const std::filesystem::path& source_path,
+                      const std::filesystem::path& dest_path)
+{
+    run_process(QStringLiteral("diskutil"),
+                QStringList() << "image" << "create" << "from"
+                              << MP_PLATFORM.path_to_qstr(source_path)
+                              << MP_PLATFORM.path_to_qstr(dest_path) << "-f"
+                              << "asif",
+                fmt::format("convert {} to ASIF format at {}", source_path, dest_path));
+}
+
+void make_sparse(const std::filesystem::path& raw_image_path, const mp::MemorySize& disk_space)
+{
+    std::error_code ec;
+    std::filesystem::resize_file(raw_image_path, disk_space.in_bytes(), ec);
+
+    if (ec)
+        throw std::runtime_error(fmt::format("Failed to resize file: {}", ec.message()));
+}
+
+std::filesystem::path convert_to_asif(const std::filesystem::path& source_path, bool destructive)
+{
+    if (is_asif_image(source_path))
+        return source_path;
+
+    mpl::info(category, "Converting {} to ASIF format", source_path);
+
+    // NO-OP if already RAW
+    auto raw_path = mp::backend::convert(source_path, "raw");
+
+    // This is often an intermediate file so we remove it, unless it came from an existing VM
+    auto intermediate_cleanup =
+        sg::make_scope_guard([intermediate = (raw_path != source_path), raw_path]() noexcept {
+            if (intermediate)
+                QFile::remove(raw_path);
+        });
+
+    auto asif_path = std::filesystem::path(source_path).replace_extension("asif");
+    auto asif_file_cleanup =
+        sg::make_scope_guard([&asif_path]() noexcept { QFile::remove(asif_path); });
+
+    create_asif_from(raw_path, asif_path);
+    asif_file_cleanup.dismiss();
+
+    return asif_path;
+}
+} // namespace
+
+namespace multipass::applevz
+{
+std::filesystem::path AppleVZUtils::convert_to_supported_format(
+    const std::filesystem::path& image_path,
+    bool destructive) const
+{
+    return macos_at_least(26, 0) ? convert_to_asif(image_path, destructive)
+                                 : backend::convert(image_path, "raw");
+}
+
+void AppleVZUtils::resize_image(const MemorySize& disk_space,
+                                const std::filesystem::path& image_path) const
+{
+    mpl::trace(category, "Resizing image to: {}", disk_space.human_readable());
+
+    is_asif_image(image_path) ? resize_asif_image(image_path, disk_space)
+                              : make_sparse(image_path, disk_space);
+
+    mpl::trace(category, "Successfully resized image: {}", image_path);
+}
+
+bool AppleVZUtils::macos_at_least(int major, int minor, int patch) const
+{
+    NSOperatingSystemVersion v{major, minor, patch};
+    return [[NSProcessInfo processInfo] isOperatingSystemAtLeastVersion:v];
+}
+} // namespace multipass::applevz

src/platform/backends/applevz/applevz_virtual_machine.cpp

@@ -15,17 +15,17 @@
  *
  */
 
+#include <applevz/applevz_utils.h>
 #include <applevz/applevz_virtual_machine.h>
-
 #include <multipass/exceptions/internal_timeout_exception.h>
 #include <multipass/exceptions/virtual_machine_state_exceptions.h>
 #include <multipass/top_catch_all.h>
+#include <multipass/utils/qemu_img_utils.h>
 #include <multipass/vm_status_monitor.h>
-
-#include <qemu/qemu_img_utils.h>
 #include <shared/macos/backend_utils.h>
 
-namespace mpl = multipass::logging;
+namespace mp = multipass;
+namespace mpl = mp::logging;
 
 namespace
 {
@@ -54,7 +54,7 @@ AppleVZVirtualMachine::~AppleVZVirtualMachine()
 
     if (vm_handle)
     {
-        multipass::top_catch_all(vm_name, [this]() {
+        mp::top_catch_all(vm_name, [this]() {
             if (state == State::running)
             {
                 suspend();
@@ -192,16 +192,16 @@ void AppleVZVirtualMachine::shutdown(ShutdownPolicy shutdown_policy)
     // We need to wait here.
     auto on_timeout = [] { throw std::runtime_error("timed out waiting for shutdown"); };
 
-    multipass::utils::try_action_for(on_timeout, std::chrono::seconds{180}, [this]() {
+    mp::utils::try_action_for(on_timeout, std::chrono::seconds{180}, [this]() {
         switch (current_state())
         {
         case VirtualMachine::State::stopped:
         case VirtualMachine::State::off:
             drop_ssh_session();
             vm_handle.reset();
-            return multipass::utils::TimeoutAction::done;
+            return mp::utils::TimeoutAction::done;
         default:
-            return multipass::utils::TimeoutAction::retry;
+            return mp::utils::TimeoutAction::retry;
         }
     });
 }
@@ -271,7 +271,7 @@ std::string AppleVZVirtualMachine::ssh_username()
 std::optional<IPAddress> AppleVZVirtualMachine::management_ipv4()
 {
     if (!management_ip)
-        management_ip = multipass::backend::get_neighbour_ip(desc.default_mac_address);
+        management_ip = mp::backend::get_neighbour_ip(desc.default_mac_address);
 
     return management_ip;
 }
@@ -296,7 +296,7 @@ void AppleVZVirtualMachine::resize_disk(const MemorySize& new_size)
 {
     assert(new_size > desc.disk_space);
 
-    multipass::backend::resize_instance_image(new_size, desc.image.image_path);
+    MP_APPLEVZ_UTILS.resize_image(new_size, desc.image.image_path);
     desc.disk_space = new_size;
 }
 
@@ -351,17 +351,17 @@ void AppleVZVirtualMachine::fetch_ip(std::chrono::milliseconds timeout)
 
     auto action = [this] {
         detect_aborted_start();
-        return ((management_ip = multipass::backend::get_neighbour_ip(desc.default_mac_address)))
-                   ? multipass::utils::TimeoutAction::done
-                   : multipass::utils::TimeoutAction::retry;
+        return ((management_ip = mp::backend::get_neighbour_ip(desc.default_mac_address)))
+                   ? mp::utils::TimeoutAction::done
+                   : mp::utils::TimeoutAction::retry;
     };
 
     auto on_timeout = [this, &timeout] {
         state = State::unknown;
         throw InternalTimeoutException{"determine IP address", timeout};
     };
 
-    multipass::utils::try_action_for(on_timeout, timeout, action);
+    mp::utils::try_action_for(on_timeout, timeout, action);
 }
 
 void AppleVZVirtualMachine::initialize_vm_handle()