Default delta updater can delta non-block resources

Replicating #886 for v2.0

See also #885

Author: jjcarstens

lib/nerves_hub/devices.ex

@@ -23,7 +23,7 @@ defmodule NervesHub.Devices do
   alias NervesHub.Repo
   alias NervesHub.TaskSupervisor, as: Tasks
 
-  @min_fwup_delta_updatable_version ">=1.6.0"
+  @min_fwup_delta_updatable_version ">=1.10.0"
 
   def get_device(device_id), do: Repo.get(Device, device_id)
   def get_device!(device_id), do: Repo.get!(Device, device_id)

lib/nerves_hub/firmwares/delta_updater/default.ex

@@ -31,31 +31,9 @@ defmodule NervesHub.Firmwares.DeltaUpdater.Default do
       )
 
     output_filename = uuid <> ".fw"
-    output = Path.join(work_dir, output_filename) |> Path.expand()
+    output_path = Path.join(work_dir, output_filename) |> Path.expand()
 
-    source_work_dir = Path.join(work_dir, "source")
-    target_work_dir = Path.join(work_dir, "target")
-    output_work_dir = Path.join(work_dir, "output")
-
-    File.mkdir_p(source_work_dir)
-    File.mkdir_p(target_work_dir)
-    File.mkdir_p(Path.join(output_work_dir, "data"))
-
-    {_, 0} = System.cmd("unzip", ["-qq", source_path, "-d", source_work_dir])
-    {_, 0} = System.cmd("unzip", ["-qq", target_path, "-d", target_work_dir])
-
-    source_rootfs = Path.join([source_work_dir, "data", "rootfs.img"])
-    target_rootfs = Path.join([target_work_dir, "data", "rootfs.img"])
-    out_rootfs = Path.join([output_work_dir, "data", "rootfs.img"])
-
-    {_, 0} =
-      System.cmd("xdelta3", ["-A", "-S", "-f", "-s", source_rootfs, target_rootfs, out_rootfs])
-
-    File.mkdir_p!(Path.dirname(output))
-    File.cp!(target_path, output)
-
-    {_, 0} = System.cmd("zip", ["-qq", output, "data/rootfs.img"], cd: output_work_dir)
-    output
+    do_delta_file(source_path, target_path, output_path, work_dir)
   end
 
   @impl NervesHub.Firmwares.DeltaUpdater
@@ -70,6 +48,73 @@ defmodule NervesHub.Firmwares.DeltaUpdater.Default do
   @impl NervesHub.Firmwares.DeltaUpdater
   def delta_updatable?(file_path) do
     {meta, 0} = System.cmd("unzip", ["-qqp", file_path, "meta.conf"])
-    meta =~ "delta-source-raw-offset" && meta =~ "delta-source-raw-count"
+
+    (meta =~ "delta-source-raw-offset" && meta =~ "delta-source-raw-count") or
+      (meta =~ "delta-source-fat-offset" && meta =~ "delta-source-fat-path")
+  end
+
+  def do_delta_file(source_path, target_path, output_path, work_dir) do
+    File.mkdir_p(work_dir)
+
+    source_work_dir = Path.join(work_dir, "source")
+    target_work_dir = Path.join(work_dir, "target")
+    output_work_dir = Path.join(work_dir, "output")
+
+    File.mkdir_p(source_work_dir)
+    File.mkdir_p(target_work_dir)
+    File.mkdir_p(output_work_dir)
+
+    {_, 0} = System.cmd("unzip", ["-qq", source_path, "-d", source_work_dir])
+    {_, 0} = System.cmd("unzip", ["-qq", target_path, "-d", target_work_dir])
+
+    _ =
+      for absolute <- Path.wildcard(target_work_dir <> "/**"), not File.dir?(absolute) do
+        path = Path.relative_to(absolute, target_work_dir)
+
+        if String.starts_with?(path, "meta.") do
+          File.cp!(Path.join(target_work_dir, path), Path.join(output_work_dir, path))
+        else
+          args = [
+            "-A",
+            "-S",
+            "-f",
+            "-s",
+            Path.join(source_work_dir, path),
+            Path.join(target_work_dir, path),
+            Path.join(output_work_dir, path)
+          ]
+
+          {_, 0} = System.cmd("xdelta3", args)
+        end
+      end
+
+    # firmware archive files order matters:
+    # 1. meta.conf.ed25519 (optional)
+    # 2. meta.conf
+    # 3. other...
+    [
+      "meta.conf.*",
+      "meta.conf",
+      "data"
+    ]
+    |> Enum.each(&add_to_zip(&1, output_work_dir, output_path))
+
+    output_path
+  end
+
+  defp add_to_zip(glob, workdir, output) do
+    workdir
+    |> Path.join(glob)
+    |> Path.wildcard()
+    |> case do
+      [] ->
+        :ok
+
+      paths ->
+        args = ["-r", "-qq", output | Enum.map(paths, &Path.relative_to(&1, workdir))]
+        {_, 0} = System.cmd("zip", args, cd: workdir)
+
+        :ok
+    end
   end
 end

test/nerves_hub/devices_test.exs

@@ -11,7 +11,7 @@ defmodule NervesHub.DevicesTest do
   alias NervesHub.Repo
   alias Ecto.Changeset
 
-  @valid_fwup_version "1.6.0"
+  @valid_fwup_version "1.10.0"
 
   setup do
     user = Fixtures.user_fixture()