chore: forward-port release-1.2

For housekeeping alone, an exandra fork is needed in order to have
access to `Repo.checkout`

Signed-off-by: Francesco Noacco <francesco.noacco@secomind.com>

Author: noaccOS

apps/astarte_housekeeping/Dockerfile

@@ -56,10 +56,8 @@ RUN apt-get update -y && \
 ARG BUILD_ENV=prod
 
 COPY --from=builder --chown=nobody:nogroup /app/_build/$BUILD_ENV/rel/astarte_housekeeping .
-COPY --from=builder --chown=nobody:nogroup /app/entrypoint.sh .
 
 # Change to non-root user
 USER nobody
 
-ENTRYPOINT ["/bin/bash", "entrypoint.sh"]
-CMD ["start"]
+CMD ["./bin/astarte_housekeeping", "start"]

apps/astarte_housekeeping/entrypoint.sh

@@ -22,6 +22,7 @@ defmodule Astarte.Housekeeping.Application do
 
   alias Astarte.DataAccess.Config, as: DataAccessConfig
   alias Astarte.Housekeeping.Config
+  alias Astarte.Housekeeping.ReleaseTasks
 
   require Logger
 
@@ -45,6 +46,7 @@ defmodule Astarte.Housekeeping.Application do
 
     # Define workers and child supervisors to be supervised
     children = [
+      ReleaseTasks,
       Astarte.HousekeepingWeb.Telemetry,
       Astarte.HousekeepingWeb.Endpoint
     ]

apps/astarte_housekeeping/lib/astarte_housekeeping/application.ex

@@ -18,13 +18,11 @@
 
 defmodule Astarte.Housekeeping.Migrator do
   @moduledoc false
-  import Ecto.Query
 
   alias Astarte.DataAccess.Consistency
   alias Astarte.DataAccess.CSystem
   alias Astarte.DataAccess.KvStore
   alias Astarte.DataAccess.Realms.Realm
-  alias Astarte.DataAccess.Repo
   alias Astarte.Events.AMQP.Vhost
   alias Astarte.Housekeeping.Realms.Queries
   alias Astarte.Housekeeping.Realms.Realm, as: HKRealm
@@ -36,8 +34,7 @@ defmodule Astarte.Housekeeping.Migrator do
   def run_astarte_keyspace_migrations do
     _ = Logger.info("Starting to migrate Astarte keyspace.", tag: "astarte_migration_started")
 
-    with :ok <- ensure_astarte_kv_store(),
-         {:ok, astarte_schema_version} <- get_astarte_schema_version() do
+    with {:ok, astarte_schema_version} <- get_astarte_schema_version() do
       migrate_astarte_keyspace_from_version(astarte_schema_version)
     end
   end
@@ -88,50 +85,6 @@ defmodule Astarte.Housekeeping.Migrator do
     end
   end
 
-  defp ensure_astarte_kv_store do
-    keyspace_name = Realm.astarte_keyspace_name()
-
-    query =
-      from t in "system_schema.tables",
-        where:
-          t.keyspace_name == ^keyspace_name and
-            t.table_name == "kv_store",
-        select: t.table_name
-
-    consistency = Consistency.domain_model(:read)
-
-    case Repo.safe_fetch_one(query, consistency: consistency) do
-      {:ok, _item} ->
-        :ok
-
-      {:error, :not_found} ->
-        create_astarte_kv_store()
-
-      {:error, reason} ->
-        Logger.warning("Error checking Astarte kv_store existence: #{inspect(reason)}.",
-          tag: "database_error"
-        )
-
-        {:error, reason}
-    end
-  end
-
-  defp create_astarte_kv_store do
-    query = """
-    CREATE TABLE #{Realm.astarte_keyspace_name()}.kv_store (
-      group varchar,
-      key varchar,
-      value blob,
-
-      PRIMARY KEY ((group), key)
-    );
-    """
-
-    with {:ok, %{rows: nil, num_rows: 1}} <- CSystem.execute_schema_change(query) do
-      :ok
-    end
-  end
-
   defp get_astarte_schema_version do
     get_schema_version(Realm.astarte_keyspace_name())
   end

apps/astarte_housekeeping/lib/astarte_housekeeping/migrator.ex

@@ -1112,6 +1112,8 @@ defmodule Astarte.Housekeeping.Realms.Queries do
   end
 
   def initialize_database do
+    Logger.info("Starting Astarte keyspace initialization")
+
     with :ok <- create_astarte_keyspace(),
          :ok <- create_realms_table(),
          :ok <- create_astarte_kv_store(),
@@ -1179,25 +1181,21 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     opts = [consistency: consistency]
 
     case Repo.query(tablets_query, [], opts) do
-      {:ok, %{num_rows: 1}} ->
+      {:ok, _} ->
+        Logger.info("Astarte keyspace initialized")
         :ok
 
-      {:ok, res} ->
-        "Unexpected ok result from database while creating astarte keyspace: #{inspect(res)}"
-        |> Logger.warning()
-
-        {:error, :astarte_keyspace_creation_failed}
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
 
       _ ->
         case Repo.query(base_query, [], opts) do
-          {:ok, %{num_rows: 1}} ->
+          {:ok, _} ->
+            Logger.info("Astarte keyspace initialized")
             :ok
 
-          {:ok, res} ->
-            "Unexpected ok result from database while creating astarte keyspace: #{inspect(res)}"
-            |> Logger.warning()
-
-            {:error, :astarte_keyspace_creation_failed}
+          {:error, %Xandra.Error{reason: :already_exists}} ->
+            :ok
 
           error ->
             error
@@ -1215,10 +1213,18 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     """
 
     consistency = Consistency.domain_model(:write)
+    opts = [consistency: consistency]
 
-    with {:ok, %{rows: nil, num_rows: 1}} <-
-           Repo.query(query, [], consistency: consistency) do
-      :ok
+    case Repo.query(query, [], opts) do
+      {:ok, _} ->
+        Logger.info("Created Astarte realms table")
+        :ok
+
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
+
+      error ->
+        error
     end
   end
 
@@ -1234,10 +1240,18 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     """
 
     consistency = Consistency.domain_model(:write)
+    opts = [consistency: consistency]
 
-    with {:ok, %{rows: nil, num_rows: 1}} <-
-           Repo.query(query, [], consistency: consistency) do
-      :ok
+    case Repo.query(query, [], opts) do
+      {:ok, _} ->
+        Logger.info("Initialized Astarte KV Store")
+        :ok
+
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
+
+      error ->
+        error
     end
   end
 
@@ -1260,19 +1274,4 @@ defmodule Astarte.Housekeeping.Realms.Queries do
 
     KvStore.insert(kv_store_map, opts)
   end
-
-  def astarte_keyspace_existing? do
-    keyspace_name = Realm.astarte_keyspace_name()
-
-    query =
-      from k in "system_schema.keyspaces",
-        where: k.keyspace_name == ^keyspace_name,
-        select: count()
-
-    consistency = Consistency.domain_model(:read)
-
-    with {:ok, count} <- Repo.safe_fetch_one(query, consistency: consistency) do
-      {:ok, count > 0}
-    end
-  end
 end

apps/astarte_housekeeping/lib/astarte_housekeeping/realms/queries.ex

@@ -18,116 +18,39 @@
 
 defmodule Astarte.Housekeeping.ReleaseTasks do
   @moduledoc false
-  alias Astarte.DataAccess
+
+  use Task, restart: :transient
+
+  alias Astarte.DataAccess.Repo
   alias Astarte.Housekeeping.Migrator
   alias Astarte.Housekeeping.Realms.Queries
 
   require Logger
 
-  @start_apps [
-    :ecto,
-    :httpoison,
-    :logger,
-    :crypto,
-    :ssl
-  ]
-
-  def init_database do
-    :ok = start_services()
-
-    case wait_connection_and_check_astarte_keyspace() do
-      {:ok, exists?} ->
-        unless exists? do
-          _ =
-            Logger.info("Astarte keyspace not found, creating it",
-              tag: "astarte_db_initialization_started"
-            )
-
-          :ok = Queries.initialize_database()
-        end
+  def start_link(_init_arg), do: Task.start_link(__MODULE__, :ensure_migrated!, [])
 
+  def ensure_migrated! do
+    case Repo.checkout(&do_ensure_migrated!/0) do
+      :ok ->
         :ok
 
-      {:error, reason} ->
-        _ =
-          Logger.error("Can't check if Astarte keyspace exists: #{inspect(reason)}",
-            tag: "astarte_db_initialization_failed"
-          )
+      error ->
+        "Error during astarte initialization: #{inspect(error)}"
+        |> Logger.error(tag: "astarte_db_migration_failed")
 
+        # Here we raise in order to notify the supervisor something went wrong
         raise "init_database failed"
     end
-
-    :ok = stop_services()
   end
 
-  def migrate do
-    :ok = start_services()
-
-    Logger.info("Starting to migrate the database", tag: "astarte_db_migration_started")
+  defp do_ensure_migrated! do
+    with :ok <- Queries.initialize_database(),
+         :ok <- Migrator.run_astarte_keyspace_migrations(),
+         :ok <- Migrator.run_realms_migrations() do
+      "Astarte database correctly initialized"
+      |> Logger.info(tag: "astarte_db_initialization_finished")
 
-    case wait_connection_and_check_astarte_keyspace() do
-      {:ok, true} ->
-        with :ok <- Migrator.run_astarte_keyspace_migrations(),
-             :ok <- Migrator.run_realms_migrations() do
-          :ok
-        else
-          {:error, reason} ->
-            Logger.error("Cannot migrate the database: #{inspect(reason)}",
-              tag: "astarte_db_migration_failed"
-            )
-
-            raise "migrate failed"
-        end
-
-        :ok = stop_services()
-
-      {:ok, false} ->
-        Logger.error("Cannot migrate the database, Astarte keyspace does not exist",
-          tag: "astarte_db_migration_failed"
-        )
-
-        raise "migrate failed"
+      :ok
     end
   end
-
-  defp wait_connection_and_check_astarte_keyspace(retries \\ 60) do
-    case Queries.astarte_keyspace_existing?() do
-      {:ok, exists?} ->
-        {:ok, exists?}
-
-      {:error, :database_connection_error} ->
-        if retries > 0 do
-          :timer.sleep(1000)
-          wait_connection_and_check_astarte_keyspace(retries - 1)
-        else
-          {:error, :database_connection_error}
-        end
-
-      {:error, reason} ->
-        {:error, reason}
-    end
-  end
-
-  defp start_services do
-    DataAccess.Config.validate!()
-    Enum.each(@start_apps, &Application.ensure_all_started/1)
-
-    # Load astarte_data_access, without starting it. This makes the application env accessible.
-    :ok = Application.ensure_loaded(:astarte_data_access)
-
-    _ = Logger.info("Starting Xandra connection to #{inspect(DataAccess.Config.xandra_nodes!())}")
-
-    {:ok, _} = Application.ensure_all_started(:astarte_data_access)
-
-    :ok
-  end
-
-  defp stop_services do
-    _ =
-      Logger.info("Astarte database correctly initialized",
-        tag: "astarte_db_initialization_finished"
-      )
-
-    :init.stop()
-  end
 end

apps/astarte_housekeeping/lib/astarte_housekeeping/release_task.ex

@@ -79,6 +79,7 @@ defmodule Astarte.Housekeeping.Mixfile do
       {:plug_cowboy, "~> 2.1"},
       {:guardian, "~> 2.3.2"},
       {:excoveralls, "~> 0.15", only: :test},
+      {:exandra, github: "noaccOS/exandra", branch: "push-zlnkysuyqwmt", override: true},
       {:pretty_log, "~> 0.1"},
       {:skogsra, "~> 2.5"},
       {:observer_cli, "~> 1.5"},

apps/astarte_housekeeping/mix.exs

@@ -19,11 +19,11 @@
   "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"},
   "ecto": {:hex, :ecto, "3.13.5", "9d4a69700183f33bf97208294768e561f5c7f1ecf417e0fa1006e4a91713a834", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df9efebf70cf94142739ba357499661ef5dbb559ef902b68ea1f3c1fabce36de"},
   "ecto_morph": {:hex, :ecto_morph, "0.1.29", "bc0b915779636bd2d30c54cad6922b3cb40f85b1d4ad59bdffd3c788d9d1f972", [:mix], [{:ecto, ">= 3.0.3", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "814bed72e3d03b278c1dfb3fbc4da37f478a37518ee54f010c1ad9254f1ca0e3"},
-  "ecto_sql": {:hex, :ecto_sql, "3.13.4", "b6e9d07557ddba62508a9ce4a484989a5bb5e9a048ae0e695f6d93f095c25d60", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2b38cf0749ca4d1c5a8bcbff79bbe15446861ca12a61f9fba604486cb6b62a14"},
+  "ecto_sql": {:hex, :ecto_sql, "3.13.5", "2f8282b2ad97bf0f0d3217ea0a6fff320ead9e2f8770f810141189d182dc304e", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aa36751f4e6a2b56ae79efb0e088042e010ff4935fc8684e74c23b1f49e25fdc"},
   "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
   "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"},
   "ex_rabbit_pool": {:git, "https://github.com/leductam/ex_rabbit_pool.git", "9951452ab51d36648b9a9d3373609e48d1379a0d", []},
-  "exandra": {:hex, :exandra, "0.16.0", "848690b915c96a80f05c0bea782fd7b0fed7963ffb449f9022939fe6ade96e0c", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:xandra, "~> 0.19.4", [hex: :xandra, repo: "hexpm", optional: false]}], "hexpm", "4b95adfdde42bd953bb80d083347ac69cd71a520213b4c2cd83cecd9b01bd919"},
+  "exandra": {:git, "https://github.com/noaccOS/exandra.git", "e2a9d023ae9fc1c04a42a56585e9bfee6479fcc5", [branch: "push-zlnkysuyqwmt"]},
   "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"},
   "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"},
   "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
@@ -60,7 +60,7 @@
   "skogsra": {:hex, :skogsra, "2.5.2", "d90e5e8425799c249c462ec0f66f16d98a11d224d0fad6222493a25beb368bc4", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: true]}], "hexpm", "89843669d40dc3b0a5ac2ad1bab24d3f17bdf8364e9f700a0fa856208579ac36"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "stream_data": {:hex, :stream_data, "1.2.0", "58dd3f9e88afe27dc38bef26fce0c84a9e7a96772b2925c7b32cd2435697a52b", [:mix], [], "hexpm", "eb5c546ee3466920314643edf68943a5b14b32d1da9fe01698dc92b73f89a9ed"},
-  "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
+  "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
   "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},

apps/astarte_housekeeping/mix.lock

@@ -86,6 +86,10 @@ defmodule Astarte.Housekeeping.MigratorTest do
     test "returns ok with missing capabilities" do
       new_realm = "realm#{System.unique_integer([:positive])}"
 
+      on_exit(fn ->
+        Database.teardown_realm_keyspace(new_realm)
+      end)
+
       realm_migrations_path =
         Application.app_dir(
           :astarte_housekeeping,