chore: forward-port release-1.3

Signed-off-by: Francesco Noacco <francesco.noacco@secomind.com>

Author: noaccOS

Dockerfile

@@ -68,11 +68,11 @@ RUN apt-get update -y && \
 ARG BUILD_ENV=prod
 ARG SERVICE
 
+ENV SERVICE_CMD="./bin/$SERVICE start"
+
 COPY --from=builder --chown=nobody:nogroup /app/_build/$BUILD_ENV/rel/$SERVICE .
-COPY --from=builder --chown=nobody:nogroup /app/entrypoint.sh .
 
 # Change to non-root user
 USER nobody
 
-ENTRYPOINT ["/bin/bash", "entrypoint.sh"]
-CMD ["start"]
+CMD $SERVICE_CMD

apps/astarte_appengine_api/entrypoint.sh

@@ -83,10 +83,9 @@ defmodule Astarte.AppEngine.API.Mixfile do
       {:guardian, "~> 2.3.2"},
       {:uuid, "~> 2.0", hex: :uuid_erl},
       # Required by :phoenix_swagger, otherwise it fails finding ex_json_schema.app
-      {:ex_json_schema, "~> 0.7"},
+      {:ex_json_schema, "~> 0.9"},
       {:current_rabbit_pool, "~> 1.1"},
       {:phoenix_swagger, "~> 0.8"},
-      {:xandra, "~> 0.13"},
       {:exandra, "~> 0.13"},
       {:typed_ecto_schema, "~> 0.4"},
       {:pretty_log, "~> 0.1"},
@@ -108,7 +107,8 @@ defmodule Astarte.AppEngine.API.Mixfile do
       # Test section
       {:excoveralls, "~> 0.15", only: :test},
       {:mox, "~> 0.5", only: :test},
-      {:mimic, "~> 1.11", only: :test}
+      {:mimic, "~> 1.11", only: :test},
+      {:ecto, "~> 3.13", override: true}
     ]
   end
 

apps/astarte_appengine_api/mix.exs

@@ -413,33 +413,21 @@ defmodule Astarte.AppEngine.APIWeb.InterfaceControllerTest do
   property "forbids deletion if `allow_unset` is false", context do
     %{realm_name: realm_name, interfaces: interfaces, device: device, auth_conn: conn} = context
 
-    valid_interfaces_to_unset =
+    invalid_interfaces_to_unset =
       interfaces
       |> Enum.filter(fn interface ->
-        allow_unset? = Enum.any?(interface.mappings, &(!&1.allow_unset))
+        doesnt_allow_unset? = Enum.any?(interface.mappings, &(!&1.allow_unset))
 
         server? = interface.ownership == :server
         property? = interface.type == :properties
 
-        property? && server? && allow_unset?
+        property? && server? && doesnt_allow_unset?
       end)
 
-    check all interface <- member_of(valid_interfaces_to_unset),
+    check all interface <- member_of(invalid_interfaces_to_unset),
               mapping_to_unset <-
                 member_of(interface.mappings |> Enum.filter(&(!&1.allow_unset))),
               path <- InterfaceUpdateGenerator.path_from_endpoint(mapping_to_unset.endpoint) do
-      if mapping_to_unset.allow_unset do
-        path_tokens = path_tokens(path)
-        expected_token = [realm_name, device.encoded_id, interface.name | path_tokens]
-
-        publish_result_ok(interface, mapping_to_unset, fn args ->
-          assert %{payload: payload, topic_tokens: topic_tokens, qos: qos} = args
-          assert topic_tokens == expected_token
-          assert qos == 2
-          assert payload == ""
-        end)
-      end
-
       request_path =
         interface_values_path(
           conn,

apps/astarte_appengine_api/test/astarte_appengine_api_web/controllers/interface_controller_test.exs

@@ -169,6 +169,7 @@ defmodule Astarte.Cases.Device do
         individual_downsampable: individual_downsampable(),
         object_downsampable: object_downsampable(),
         properties_server_allow_unset: properties_server_allow_unset(),
+        properties_server_without_unset: properties_server_without_unset(),
         explicit_timestamps: explicit_timestamps(),
         other_interfaces: other_interfaces()
       }
@@ -231,6 +232,14 @@ defmodule Astarte.Cases.Device do
     |> customize_mappings(allow_unset: true)
   end
 
+  defp properties_server_without_unset do
+    [ownership: :server, type: :properties]
+    |> InterfaceGenerator.interface()
+    |> list_of(min_length: 1)
+    |> Enum.at(0)
+    |> customize_mappings(allow_unset: false)
+  end
+
   defp fallible_interfaces do
     [ownership: :server]
     |> InterfaceGenerator.interface()

apps/astarte_appengine_api/test/support/cases/device.ex

@@ -22,6 +22,7 @@ defmodule Astarte.Housekeeping.Application do
 
   alias Astarte.DataAccess.Config, as: DataAccessConfig
   alias Astarte.Housekeeping.Config
+  alias Astarte.Housekeeping.ReleaseTasks
 
   require Logger
 
@@ -45,6 +46,7 @@ defmodule Astarte.Housekeeping.Application do
 
     # Define workers and child supervisors to be supervised
     children = [
+      ReleaseTasks,
       Astarte.HousekeepingWeb.Telemetry,
       Astarte.HousekeepingWeb.Endpoint
     ]

apps/astarte_data_updater_plant/entrypoint.sh

@@ -18,13 +18,11 @@
 
 defmodule Astarte.Housekeeping.Migrator do
   @moduledoc false
-  import Ecto.Query
 
   alias Astarte.DataAccess.Consistency
   alias Astarte.DataAccess.CSystem
   alias Astarte.DataAccess.KvStore
   alias Astarte.DataAccess.Realms.Realm
-  alias Astarte.DataAccess.Repo
   alias Astarte.Events.AMQP.Vhost
   alias Astarte.Housekeeping.Realms.Queries
   alias Astarte.Housekeeping.Realms.Realm, as: HKRealm
@@ -36,8 +34,7 @@ defmodule Astarte.Housekeeping.Migrator do
   def run_astarte_keyspace_migrations do
     _ = Logger.info("Starting to migrate Astarte keyspace.", tag: "astarte_migration_started")
 
-    with :ok <- ensure_astarte_kv_store(),
-         {:ok, astarte_schema_version} <- get_astarte_schema_version() do
+    with {:ok, astarte_schema_version} <- get_astarte_schema_version() do
       migrate_astarte_keyspace_from_version(astarte_schema_version)
     end
   end
@@ -88,50 +85,6 @@ defmodule Astarte.Housekeeping.Migrator do
     end
   end
 
-  defp ensure_astarte_kv_store do
-    keyspace_name = Realm.astarte_keyspace_name()
-
-    query =
-      from t in "system_schema.tables",
-        where:
-          t.keyspace_name == ^keyspace_name and
-            t.table_name == "kv_store",
-        select: t.table_name
-
-    consistency = Consistency.domain_model(:read)
-
-    case Repo.safe_fetch_one(query, consistency: consistency) do
-      {:ok, _item} ->
-        :ok
-
-      {:error, :not_found} ->
-        create_astarte_kv_store()
-
-      {:error, reason} ->
-        Logger.warning("Error checking Astarte kv_store existence: #{inspect(reason)}.",
-          tag: "database_error"
-        )
-
-        {:error, reason}
-    end
-  end
-
-  defp create_astarte_kv_store do
-    query = """
-    CREATE TABLE #{Realm.astarte_keyspace_name()}.kv_store (
-      group varchar,
-      key varchar,
-      value blob,
-
-      PRIMARY KEY ((group), key)
-    );
-    """
-
-    with {:ok, %{rows: nil, num_rows: 1}} <- CSystem.execute_schema_change(query) do
-      :ok
-    end
-  end
-
   defp get_astarte_schema_version do
     get_schema_version(Realm.astarte_keyspace_name())
   end

apps/astarte_housekeeping/entrypoint.sh

@@ -218,7 +218,7 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     if opts[:async] do
       {:ok, _pid} =
         Task.start(fn ->
-          do_create_realm(
+          do_execute_realm_creation(
             realm_name,
             keyspace_name,
             public_key_pem,
@@ -230,7 +230,7 @@ defmodule Astarte.Housekeeping.Realms.Queries do
 
       {:ok, :started}
     else
-      do_create_realm(
+      do_execute_realm_creation(
         realm_name,
         keyspace_name,
         public_key_pem,
@@ -241,6 +241,26 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     end
   end
 
+  defp do_execute_realm_creation(
+         realm_name,
+         keyspace_name,
+         public_key_pem,
+         replication_map_str,
+         device_limit,
+         max_retention
+       ) do
+    Repo.checkout(fn ->
+      do_create_realm(
+        realm_name,
+        keyspace_name,
+        public_key_pem,
+        replication_map_str,
+        device_limit,
+        max_retention
+      )
+    end)
+  end
+
   defp do_create_realm(
          realm_name,
          keyspace_name,
@@ -1127,6 +1147,8 @@ defmodule Astarte.Housekeeping.Realms.Queries do
   end
 
   def initialize_database do
+    Logger.info("Starting Astarte keyspace initialization")
+
     with :ok <- create_astarte_keyspace(),
          :ok <- create_realms_table(),
          :ok <- create_astarte_kv_store(),
@@ -1194,25 +1216,21 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     opts = [consistency: consistency]
 
     case Repo.query(tablets_query, [], opts) do
-      {:ok, %{num_rows: 1}} ->
+      {:ok, _} ->
+        Logger.info("Astarte keyspace initialized")
         :ok
 
-      {:ok, res} ->
-        "Unexpected ok result from database while creating astarte keyspace: #{inspect(res)}"
-        |> Logger.warning()
-
-        {:error, :astarte_keyspace_creation_failed}
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
 
       _ ->
         case Repo.query(base_query, [], opts) do
-          {:ok, %{num_rows: 1}} ->
+          {:ok, _} ->
+            Logger.info("Astarte keyspace initialized")
             :ok
 
-          {:ok, res} ->
-            "Unexpected ok result from database while creating astarte keyspace: #{inspect(res)}"
-            |> Logger.warning()
-
-            {:error, :astarte_keyspace_creation_failed}
+          {:error, %Xandra.Error{reason: :already_exists}} ->
+            :ok
 
           error ->
             error
@@ -1230,10 +1248,18 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     """
 
     consistency = Consistency.domain_model(:write)
+    opts = [consistency: consistency]
 
-    with {:ok, %{rows: nil, num_rows: 1}} <-
-           Repo.query(query, [], consistency: consistency) do
-      :ok
+    case Repo.query(query, [], opts) do
+      {:ok, _} ->
+        Logger.info("Created Astarte realms table")
+        :ok
+
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
+
+      error ->
+        error
     end
   end
 
@@ -1249,10 +1275,18 @@ defmodule Astarte.Housekeeping.Realms.Queries do
     """
 
     consistency = Consistency.domain_model(:write)
+    opts = [consistency: consistency]
 
-    with {:ok, %{rows: nil, num_rows: 1}} <-
-           Repo.query(query, [], consistency: consistency) do
-      :ok
+    case Repo.query(query, [], opts) do
+      {:ok, _} ->
+        Logger.info("Initialized Astarte KV Store")
+        :ok
+
+      {:error, %Xandra.Error{reason: :already_exists}} ->
+        :ok
+
+      error ->
+        error
     end
   end
 
@@ -1275,19 +1309,4 @@ defmodule Astarte.Housekeeping.Realms.Queries do
 
     KvStore.insert(kv_store_map, opts)
   end
-
-  def astarte_keyspace_existing? do
-    keyspace_name = Realm.astarte_keyspace_name()
-
-    query =
-      from k in "system_schema.keyspaces",
-        where: k.keyspace_name == ^keyspace_name,
-        select: count()
-
-    consistency = Consistency.domain_model(:read)
-
-    with {:ok, count} <- Repo.safe_fetch_one(query, consistency: consistency) do
-      {:ok, count > 0}
-    end
-  end
 end