Added CI and updated OTP source trees

Author: dominicletz

.dockerignore

@@ -1,3 +1,11 @@
 /otp
-/_build
-*.zip
+/_build/dev
+/_build/x86_64
+/_build/arm
+/_build/arm64
+/lib
+/.history
+/.elixir_ls
+/.github
+*.zip
+*.tmp

.github/workflows/ci.yml

@@ -1,5 +1,7 @@
 name: "CI"
 on: ["push", "pull_request"]
+env:
+  OTP_TAG: ios
 
 jobs:
   build:

README.md

@@ -5,16 +5,22 @@ To use elixir-desktop on mobile-phones this projects packages the BEAM Virtual M
 - Android arm 64-bit
 - Android arm 32-bit
 - Android x86 64-bit (for the Android Simulator)
+- iOS arm 32-bit (very old iPhones)
+- iOS arm 64-bit (current iPhones)
+- iOS arm 64-bit (MacOS M1 Simulator)
+- iOS x86_64     (MacOS Intel Simulator)
 
 ## Building
 
-Runtimes depends on docker and the dockercross/* docker-images created for cross-compilation. If docker is installed for your
-current user then building all the runtimes bundled in a zip file is as easy as:
+Android runtimes depends on docker and the dockercross/* docker-images created for cross-compilation. If docker is installed for your current user then building all the runtimes bundled in a zip file is as easy as:
 
-`mix package.runtime`
+`mix package.android.runtime`
 
 After this you should have all runtimes in `_build/#{arch}-runtime.zip` these then will need to be packaged with your mobile app. 
 
+For iOS builds are triggered similiary with:
+
+`mix package.ios.runtime`
 
 ## Android Versions and API-Levels (update Sept. 7th 2021)
 

lib/beam.dockerfile renamed to lib/mix/tasks/android_beam.dockerfile

@@ -5,7 +5,7 @@ ENV NDK_ROOT $CROSS_ROOT
 ENV ANDROID_NDK_HOME $CROSS_ROOT
 ENV NDK_ABI_PLAT <%= @arch.android_name %><%= @arch.abi %>
 ENV PATH $NDK_ROOT/bin:$PATH
-ENV FC= CPP= LD= CC=clang AR=ar
+ENV FC= CPP= LD= CXX=clang++ CC=clang AR=ar
 ENV MAKEFLAGS "-j10 -O"
 
 # Setting up openssl
@@ -19,7 +19,7 @@ RUN cp ${NDK_ROOT}/bin/llvm-ranlib ${NDK_ROOT}/bin/<%= @arch.cpu %>-linux-<%= @a
 RUN ARCH="android-<%= @arch.id %> -D__ANDROID_API__=<%= @arch.abi %>" ./install_openssl.sh
 
 # Fetching OTP
-RUN git clone --depth 1 -b diode/beta https://github.com/diodechain/otp.git
+COPY _build/otp otp
 
 <%= if @arch.id == "arm" do %>
 ENV LIBS -L$NDK_ROOT/lib64/clang/12.0.5/lib/linux/ /usr/local/openssl/lib/libcrypto.a -lclang_rt.builtins-arm-android
@@ -30,7 +30,8 @@ ENV LIBS /usr/local/openssl/lib/libcrypto.a
 # We need -z global for liberlang.so because:
 # https://android-ndk.narkive.com/iNWj05IV/weak-symbol-linking-when-loading-dynamic-libraries
 # https://android.googlesource.com/platform/bionic/+/30b17e32f0b403a97cef7c4d1fcab471fa316340/linker/linker_namespaces.cpp#100
-ENV CFLAGS="-Os -fPIC" CXXFLAGS="-Os -fPIC" LDFLAGS="-z global"
+ENV CFLAGS="-Os -fPIC" CXXFLAGS="-Os -fPIC" LDFLAGS="-z global" CXX= CC=
+
 # RUN env
 WORKDIR /work/otp
 RUN ./otp_build autoconf
@@ -41,7 +42,9 @@ RUN ./otp_build autoconf
 # Build run #1, building the x86 based cross compiler which will generate the .beam files
 <% 
 config = "--with-ssl=/usr/local/openssl/ --disable-dynamic-ssl-lib --without-javac --without-odbc --without-wx --without-debugger --without-observer --without-cdv --without-et --xcomp-conf=xcomp/erl-xcomp-#{@arch.id}-android.conf"
-config = if @arch.id == "x86_64", do: "--disable-jit #{config}", else: config
+# Disabled jit for arm and x86_64 until https://github.com/erlang/otp/issues/4950 is fixed
+# config = if @arch.id == "x86_64", do: "--disable-jit #{config}", else: config
+config = "--disable-jit #{config}"
 %>
 RUN ./otp_build configure <%= config %>
 RUN ./otp_build boot -a

lib/nif.dockerfile renamed to lib/mix/tasks/android_nif.dockerfile

@@ -11,15 +11,16 @@ RUN apt install -y erlang
 WORKDIR /work
 RUN mix local.hex --force && mix local.rebar
 
-ENV ERLANG_PATH /work/otp/release/<%= @arch.pc %>-linux-<%= @arch.android_name %>/erts-12.0/include
-ENV ERTS_INCLUDE_DIR /work/otp/release/<%= @arch.pc %>-linux-<%= @arch.android_name %>/erts-12.0/include
+ENV ERLANG_PATH /work/otp/release/<%= @arch.pc %>-linux-<%= @arch.android_name %>/erts-<%= @erts_version %>/include
+ENV ERTS_INCLUDE_DIR /work/otp/release/<%= @arch.pc %>-linux-<%= @arch.android_name %>/erts-<%= @erts_version %>/include
 ENV HOST <%= @arch.cpu %>
 ENV CROSSCOMPILE Android
+ENV CC=clang CXX=clang++
 
-RUN git clone     <%= @repo %>
+RUN git clone <%= @repo %>
 WORKDIR /work/<%= @basename %>
 <%= if @tag do %> 
-RUN git checkout @tag
+RUN git checkout <%= @tag %>
 <% end %>
 
 # Three variants of building {:mix, :make, :rebar3}

lib/mix/tasks/package_android_nif.ex

@@ -1,10 +1,11 @@
 defmodule Mix.Tasks.Package.Android.Nif do
   use Mix.Task
+  alias Mix.Tasks.Package.Android.Runtime
   require EEx
 
   def run([]) do
     for nif <- Runtimes.default_nifs() do
-      for arch <- Runtimes.default_archs() do
+      for arch <- Runtime.default_archs() do
         build(arch, Runtimes.get_nif(nif))
       end
     end
@@ -22,7 +23,7 @@ defmodule Mix.Tasks.Package.Android.Nif do
   end
 
   defp build(arch, nif) do
-    type = Runtimes.get_arch(arch).android_type
+    type = Runtime.get_arch(arch).android_type
     target = "_build/#{type}-nif-#{nif.name}.zip"
 
     if exists?(target) do
@@ -32,7 +33,7 @@ defmodule Mix.Tasks.Package.Android.Nif do
 
       Runtimes.docker_build(
         image_name,
-        Runtimes.generate_nif_dockerfile(arch, nif)
+        Runtime.generate_nif_dockerfile(arch, nif)
       )
 
       Runtimes.run(~w(docker run --rm

lib/mix/tasks/package_android_runtime.ex

@@ -3,21 +3,23 @@ defmodule Mix.Tasks.Package.Android.Runtime do
   require EEx
 
   def run(_) do
-    buildall(Map.keys(Runtimes.architectures()))
+    buildall(default_archs())
   end
 
   defp target(arch) do
     "_build/#{arch.android_type}-runtime.zip"
   end
 
   def build(arch) do
-    arch = Runtimes.get_arch(arch)
+    arch = get_arch(arch)
     file_name = target(arch)
 
     if File.exists?(file_name) do
       :ok
     else
-      {content, _args} = Runtimes.generate_beam_dockerfile(arch.id)
+      Runtimes.ensure_otp()
+
+      {content, _args} = generate_beam_dockerfile(arch.id)
       image_name = "beam-#{arch.id}"
       file = "#{image_name}.dockerfile.tmp"
       File.write!(file, content)
@@ -47,4 +49,67 @@ defmodule Mix.Tasks.Package.Android.Runtime do
     build(head)
     buildall(rest)
   end
+
+  def architectures() do
+    %{
+      "arm" => %{
+        id: "arm",
+        abi: 23,
+        cpu: "arm",
+        pc: "arm-unknown",
+        android_name: "androideabi",
+        android_type: "armeabi-v7a"
+      },
+      "arm64" => %{
+        id: "arm64",
+        abi: 23,
+        cpu: "aarch64",
+        pc: "aarch64-unknown",
+        android_name: "android",
+        android_type: "arm64-v8a"
+      },
+      "x86_64" => %{
+        id: "x86_64",
+        abi: 23,
+        cpu: "x86_64",
+        pc: "x86_64-pc",
+        android_name: "android",
+        android_type: "x86_64"
+      }
+    }
+  end
+
+  def get_arch(arch) do
+    Map.fetch!(architectures(), arch)
+  end
+
+  def generate_beam_dockerfile(arch) do
+    args = [arch: get_arch(arch), erts_version: Runtimes.erts_version()]
+    {beam_dockerfile(args), args}
+  end
+
+  def generate_nif_dockerfile(arch, nif) do
+    {parent, args} = generate_beam_dockerfile(arch)
+
+    args =
+      args ++
+        [parent: parent, repo: nif.repo, tag: nif.tag, basename: nif.basename]
+
+    content = nif_dockerfile(args)
+
+    file = "#{nif.basename}-#{arch}.dockerfile.tmp"
+    File.write!(file, content)
+    file
+  end
+
+  EEx.function_from_file(:defp, :nif_dockerfile, "#{__DIR__}/android_nif.dockerfile", [:assigns])
+
+  EEx.function_from_file(:defp, :beam_dockerfile, "#{__DIR__}/android_beam.dockerfile", [:assigns])
+
+  def default_archs() do
+    case System.get_env("ARCH", nil) do
+      nil -> ["arm", "arm64", "x86_64"]
+      arch -> [arch]
+    end
+  end
 end

lib/mix/tasks/package_ios_nif.ex

@@ -48,11 +48,14 @@ defmodule Mix.Tasks.Package.Ios.Nif do
     sdkroot = String.trim(sdkroot)
     cflags = arch.cflags <> " -isysroot #{sdkroot} -I#{Path.absname("stubs")}"
     lflags = "-Wl,-syslibroot,#{sdkroot} -lc++"
+    erts_version = Runtimes.erts_version()
 
     env = [
       PATH: path,
-      ERLANG_PATH: Path.join(otp_target(arch), "release/#{arch.name}/erts-12.0/include"),
-      ERTS_INCLUDE_DIR: Path.join(otp_target(arch), "release/#{arch.name}/erts-12.0/include"),
+      ERLANG_PATH:
+        Path.join(otp_target(arch), "release/#{arch.name}/erts-#{erts_version}/include"),
+      ERTS_INCLUDE_DIR:
+        Path.join(otp_target(arch), "release/#{arch.name}/erts-#{erts_version}/include"),
       HOST: arch.name,
       CROSSCOMPILE: "iOS",
       STATIC_ERLANG_NIF: "yes",

lib/mix/tasks/package_ios_runtime.ex

@@ -1,7 +1,4 @@
 defmodule Mix.Tasks.Package.Ios.Runtime do
-  @otp_source "https://github.com/diodechain/otp"
-  @otp_tag "diode/ios"
-
   alias Mix.Tasks.Package.Ios.Nif
   use Mix.Task
   require EEx
@@ -97,7 +94,7 @@ defmodule Mix.Tasks.Package.Ios.Runtime do
       IO.puts("liberlang.a (#{arch.id}) already exists...")
     else
       if !File.exists?(otp_target(arch)) do
-        ensure_otp()
+        Runtimes.ensure_otp()
         Runtimes.run(~w(git clone _build/otp #{otp_target(arch)}))
       end
 
@@ -198,7 +195,7 @@ defmodule Mix.Tasks.Package.Ios.Runtime do
   end
 
   defp buildall(targets, nifs) do
-    ensure_otp()
+    Runtimes.ensure_otp()
 
     # targets
     # |> Enum.map(fn target -> Task.async(fn -> build(target, nifs) end) end)
@@ -239,14 +236,4 @@ defmodule Mix.Tasks.Package.Ios.Runtime do
     Runtimes.run("lipo -create #{Enum.join(more, " ")} -output #{tmp}")
     [tmp]
   end
-
-  defp ensure_otp() do
-    if !File.exists?("_build/otp") do
-      File.mkdir_p!("_build")
-
-      Runtimes.run(
-        "git clone #{@otp_source} _build/otp && cd _build/otp && git checkout #{@otp_tag}"
-      )
-    end
-  end
 end