Fixes for preview environments (#2191)

Author: michaeljguarino

lib/console.ex

@@ -4,6 +4,7 @@ defmodule Console do
   @ttl :timer.minutes(30)
 
   @type error :: {:error, term}
+  @cache Application.compile_env(:console, :cache_adapter)
 
   def version(), do: conf(:version)
 
@@ -16,14 +17,30 @@ defmodule Console do
   def coalesce(val, _), do: val
 
   def debounce(scope, fun, opts \\ []) do
-    case conf(:cache_adapter).get({:plrl_debounce, scope}) do
+    case @cache.get({:plrl_debounce, scope}) do
       nil ->
-        conf(:cache_adapter).put({:plrl_debounce, scope}, opts[:placeholder] || :ok, opts ++ [ttl: @ttl])
+        @cache.put({:plrl_debounce, scope}, opts[:placeholder] || :ok, opts ++ [ttl: @ttl])
         fun.()
       res -> res
     end
   end
 
+  @doc """
+  Executes a function only once, even if the function is called multiple times, within a given time frame.
+
+  If ignored, just returns ok
+  """
+  @spec nonce(term, term, (() -> term), keyword()) :: term | :ok
+  def nonce(scope, input, fun, opts \\ []) when is_function(fun, 0) do
+    sha = :erlang.phash2(input)
+    case @cache.get({:plrl_nonce, scope}) do
+      ^sha -> :ok
+      _ ->
+        @cache.put({:plrl_nonce, scope}, sha, opts ++ [ttl: @ttl])
+        fun.()
+    end
+  end
+
   def rate_limit(), do: {"global", :timer.seconds(1), Console.conf(:qps)}
 
   def provider(), do: Console.conf(:provider)

lib/console/ai/chat.ex

@@ -226,12 +226,13 @@ defmodule Console.AI.Chat do
       |> ChatThread.changeset(Map.merge(Console.mapify(thread), %{summary: "Clone of #{thread.summary}"}))
       |> Repo.insert()
     end)
-    |> add_operation(:chats, fn %{thread: thread} ->
-      Chat.for_thread(thread_id)
+    |> add_operation(:chats, fn %{thread: thread, clone: clone} ->
+      Chat.for_thread(thread.id)
       |> Chat.before(seq)
       |> Repo.all()
+      |> Enum.map(&Map.take(&1, ~w(content role confirm confirmed_at attributes server_id pull_request_id)a))
       |> Enum.map(&Console.mapify/1)
-      |> save_messages(thread.id, user)
+      |> save_messages(clone.id, user)
     end)
     |> execute(extract: :clone)
   end

lib/console/deployments/flows/preview.ex

@@ -45,37 +45,49 @@ defmodule Console.Deployments.Flows.Preview do
     end
   end
 
-  def sync_service(%Service{} = svc) do
+  def fresh?(updated_at), do: Timex.after?(updated_at || Timex.now(), Timex.shift(Timex.now(), minutes: -1))
+
+  def sync_service(%Service{deleted_at: nil} = svc) do
     case Repo.preload(svc, [:preview_instance, :preview_templates]) do
       %Service{preview_instance: %PreviewEnvironmentInstance{} = inst} ->
         post_comment(inst)
-      %Service{preview_templates: [_ | _], id: id} ->
-        PreviewEnvironmentInstance.for_service(id)
-        |> PreviewEnvironmentInstance.preloaded()
-        |> PreviewEnvironmentInstance.ordered(asc: :id)
-        |> Repo.stream(method: :keyset)
-        |> Console.throttle()
-        |> Enum.each(&update_instance(&1, &1.pull_request))
+      %Service{preview_templates: [_ | _], id: id, updated_at: updated_at} ->
+        if fresh?(updated_at) do
+          PreviewEnvironmentInstance.for_service(id)
+          |> PreviewEnvironmentInstance.preloaded()
+          |> PreviewEnvironmentInstance.ordered(asc: :id)
+          |> Repo.stream(method: :keyset)
+          |> Console.throttle()
+          |> Enum.each(&update_instance(&1, &1.pull_request))
+        end
       _ -> :ok
     end
   end
+  def sync_service(_), do: :ok
 
   defp post_comment(%PreviewEnvironmentInstance{} = inst) do
     %PreviewEnvironmentInstance{
-      pull_request: %PullRequest{url: url} = pr,
+      pull_request: %PullRequest{url: url, commit_sha: sha} = pr,
       service: svc,
       template: %PreviewEnvironmentTemplate{comment_template: msg} = tpl,
-    } = inst = Repo.preload(inst, [:service, :template, :pull_request])
+    } = inst = Repo.preload(inst, [:template, :pull_request, service: :cluster])
 
     with %ScmConnection{} = conn <- scm_connection(tpl),
          message = EEx.eval_string(@preview_comment, assigns: [
            url: Console.url("/cd/clusters/#{svc.cluster_id}/services/#{svc.id}/components"),
            comment_template: msg,
            service: svc,
-           status_color: color(svc.status)
+           sha: sha,
+           status_color: color(svc.status),
+           status_emoji: emoji(svc.status),
+           status_text: status(svc.status),
+           logs_url: Console.url("/cd/clusters/#{svc.cluster_id}/services/#{svc.id}/logs"),
+           flow_url: Console.url("/flows/#{tpl.flow_id}/services")
          ]),
          {:ok, message} <- render_liquid(message, liquid_context(pr, tpl)) do
-      Pr.Dispatcher.review(conn, %{pr | comment_id: Console.deep_get(inst, ~w(status comment_id)a)}, message)
+      Console.nonce({:svc_review, svc.id}, message, fn ->
+        Pr.Dispatcher.review(conn, %{pr | comment_id: Console.deep_get(inst, ~w(status comment_id)a)}, message)
+      end, ttl: :timer.seconds(15))
     else
       _ -> {:error, "could not create review comment for pr: #{url}"}
     end
@@ -90,7 +102,7 @@ defmodule Console.Deployments.Flows.Preview do
   def delete_instance(_), do: :ok
 
   defp create_instance(
-    %PreviewEnvironmentTemplate{reference_service: ref} = template,
+    %PreviewEnvironmentTemplate{reference_service: %Service{} = ref} = template,
     %PullRequest{} = pr
   ) do
     with {:ok, attrs} <- build_attributes(pr, template),
@@ -123,7 +135,9 @@ defmodule Console.Deployments.Flows.Preview do
     with {:ok, ns}   <- render_liquid(tpl.namespace || svc.namespace, ctx),
          {:ok, name} <- render_liquid(tpl.name || svc.name, ctx),
          {:ok, tpl}  <- template_helm_vals(svc,tpl, ctx) do
-      {:ok, ServiceTemplate.attributes(tpl, ns, name)}
+      ServiceTemplate.attributes(tpl, ns, name)
+      |> drop_nils_recursive()
+      |> ok()
     end
   end
 
@@ -148,33 +162,58 @@ defmodule Console.Deployments.Flows.Preview do
   defp scm_connection(_), do: Git.default_scm_connection()
 
   defp template_helm_vals(%Service{} = svc, %ServiceTemplate{helm: %{values: v}} = tpl, ctx) do
-    with {:ok, svc_values} <- Console.deep_get(svc, ~w(helm values)a, "{}") |> YamlElixir.read_from_string(),
-         {:ok, new_str}    <- render_liquid(v, ctx),
-         {:ok, new_values} <- YamlElixir.read_from_string(new_str),
-         {:ok, vals}       <- DeepMerge.deep_merge(svc_values, new_values) |> Jason.encode(),
+    with {:ok, svc_values}  <- safe_yaml(Console.deep_get(svc, ~w(helm values)a)),
+         {:ok, tpl_values}  <- render_liquid(v, ctx),
+         {:ok, new_values}  <- safe_yaml(tpl_values),
+         {:ok, vals}        <- DeepMerge.deep_merge(svc_values, new_values) |> Jason.encode(),
       do: {:ok, put_in(tpl.helm.values, vals)}
   end
   defp template_helm_vals(_, %ServiceTemplate{} = tpl, _), do: {:ok, tpl}
 
-  defp render_liquid(template, ctx) do
+  defp safe_yaml(val) when is_binary(val), do: YamlElixir.read_from_string(val)
+  defp safe_yaml(nil), do: {:ok, %{}}
+
+  defp render_liquid(template, ctx) when is_binary(template) do
     with {:parse, {:ok, tpl}} <- {:parse, Solid.parse(template)},
-         {:render, {:ok, res}} <- {:render, Solid.render(tpl, ctx, strict_variables: true)} do
+         {:render, {:ok, res, _}} <- {:render, Solid.render(tpl, ctx, strict_variables: true)} do
       {:ok, IO.iodata_to_binary(res)}
     else
-      {:parse, {:error, %Solid.TemplateError{message: message}}} -> {:error, message}
+      {:parse, {:error, %Solid.TemplateError{} = err}} -> {:error, Solid.TemplateError.message(err)}
       {:render, {:error, errs, _}} -> {:error, Enum.map(errs, &inspect/1) |> Enum.join(", ")}
     end
   end
+  defp render_liquid(template, _), do: {:ok, template}
 
   defp color(:healthy), do: :green
   defp color(:failed), do: :red
   defp color(_), do: :yellow
 
+  defp emoji(:healthy), do: ":white_check_mark:"
+  defp emoji(:failed), do: ":x:"
+  defp emoji(_), do: ":wrench:"
+
+  defp status(:healthy), do: "ready!"
+  defp status(:failed), do: "failed to deploy (check Plural to see details)"
+  defp status(_), do: "building..."
+
   defp bot(), do: %{Users.get_bot!("console") | roles: %{admin: true}}
 
   defp notify({:ok, %PreviewEnvironmentInstance{} = inst}, :create),
     do: handle_notify(PubSub.PreviewEnvironmentInstanceCreated, inst)
   defp notify({:ok, %PreviewEnvironmentInstance{} = inst}, :update),
     do: handle_notify(PubSub.PreviewEnvironmentInstanceUpdated, inst)
   defp notify(pass, _), do: pass
+
+  defp drop_nils_recursive(%{__struct__: _} = struct) do
+    Map.from_struct(struct)
+    |> drop_nils_recursive()
+  end
+
+  defp drop_nils_recursive(%{} = map) do
+    Enum.filter(map, &elem(&1, 1) != nil)
+    |> Map.new(fn {k, v} -> {k, drop_nils_recursive(v)} end)
+  end
+
+  defp drop_nils_recursive(list) when is_list(list), do: Enum.map(list, &drop_nils_recursive/1)
+  defp drop_nils_recursive(v), do: v
 end

lib/console/deployments/observer/poller.ex

@@ -43,7 +43,7 @@ defmodule Console.Deployments.Observer.Poller do
   defp poll_oci(%{oci: %{url: url} = oci} = target, last) do
     client = OCI.Client.new(url)
     with {:ok, oci} <- OCI.Auth.authenticate(client, oci.provider, oci.auth),
-         {:ok, %OCI.Tags{tags: [_ | _] = tags}} <- OCI.Client.tags(oci),
+         {:ok, %OCI.Tags{tags: [_ | _] = tags}} <- OCI.Client.tags(oci, &is_semver?/1),
          {:tag, [vsn | _]} <- {:tag, sorted(tags, target)},
          {:vsn, :gt} <- {:vsn, compare(vsn, last, target)} do
       {:ok, vsn}

lib/console/deployments/pr/utils.ex

@@ -13,6 +13,8 @@ defmodule Console.Deployments.Pr.Utils do
   @flow_regex [~r/plrl\/flow\/([[:alnum:]_\-]+)\/?/, ~r/plrl\(flow:([[:alnum:]_\-]*)\)/, ~r/Plural [fF]low:\s+([[:alnum:]_\-]+)/]
   @preview_regex [~r/plrl\/preview\/([[:alnum:]_\-]+)\/?/, ~r/plrl\(preview:([[:alnum:]_\-]*)\)/, ~r/Plural [pP]review:\s+([[:alnum:]_\-]+)/]
 
+  @solid_opts [strict_variables: true, strict_filters: true]
+
   def filter_ansi(text), do: String.replace(text, @ansi_code, "")
 
   def pr_associations(content, scopes \\ ~w(stack service cluster flow)a) do
@@ -64,20 +66,20 @@ defmodule Console.Deployments.Pr.Utils do
 
   def render_solid_raw(template, ctx) do
     with {:parse, {:ok, tpl}} <- {:parse, Solid.parse(template)},
-         {:render, {:ok, res}} <- {:render, Solid.render(tpl, %{"context" => ctx})} do
+         {:render, {:ok, res, _}} <- {:render, Solid.render(tpl, %{"context" => ctx}, @solid_opts)} do
       {:ok, IO.iodata_to_binary(res)}
     else
-      {:parse, {:error, %Solid.TemplateError{message: message}}} -> {:error, message}
+      {:parse, {:error, %Solid.TemplateError{} = err}} -> {:error, Solid.TemplateError.message(err)}
       {:render, {:error, errs, _}} -> {:error, Enum.map(errs, &inspect/1) |> Enum.join(", ")}
     end
   end
 
   def render_solid(template, ctx) do
     with {:parse, {:ok, tpl}} <- {:parse, Solid.parse(template)},
-         {:render, {:ok, res}} <- {:render, Solid.render(tpl, %{"context" => ctx})} do
+         {:render, {:ok, res, _}} <- {:render, Solid.render(tpl, %{"context" => ctx}, @solid_opts)} do
       {:ok, IO.iodata_to_binary(res)}
     else
-      {:parse, {:error, %Solid.TemplateError{message: message}}} -> {:error, message}
+      {:parse, {:error, %Solid.TemplateError{} = err}} -> {:error, Solid.TemplateError.message(err)}
       {:render, {:error, errs, _}} -> {:error, "encountered #{length(errs)} while rendering pr description"}
     end
   end

lib/console/deployments/pubsub/protocols/previewable.ex

@@ -17,29 +17,32 @@ defimpl Console.Deployments.PubSub.Previewable, for: [
   Console.PubSub.PullRequestUpdated
 ] do
   alias Console.Schema.PullRequest
+  alias Console.Deployments.Notifications.Utils
   alias Console.Deployments.Flows.Preview
 
   def reconcile(%@for{item: %PullRequest{status: s} = pr}) when s in ~w(merged closed)a,
     do: Preview.delete_instance(pr)
-  def reconcile(%@for{item: %PullRequest{} = pr}), do: Preview.sync_instance(pr)
+  def reconcile(%@for{item: %PullRequest{} = pr}) do
+    Utils.deduplicate({:pr_updated, pr.id}, fn ->
+      Preview.sync_instance(pr)
+    end, ttl: :timer.seconds(15))
+  end
 end
 
-defimpl Console.Deployments.PubSub.Previewable, for: [
-  Console.PubSub.PreviewEnvironmentInstanceCreated,
-] do
+defimpl Console.Deployments.PubSub.Previewable, for: Console.PubSub.PreviewEnvironmentInstanceCreated do
   alias Console.Deployments.Flows.Preview
 
   def reconcile(%@for{item: inst}), do: Preview.pr_comment(inst)
 end
 
 defimpl Console.Deployments.PubSub.Previewable, for: [
   Console.PubSub.ServiceUpdated,
+  Console.PubSub.ServiceComponentsUpdated
 ] do
   alias Console.Deployments.Flows.Preview
 
   def reconcile(%@for{item: svc}) do
-    fresh = Timex.now() |> Timex.shift(seconds: -10)
-    case Timex.after?(svc.updated_at || Timex.now(), fresh) do
+    case Preview.fresh?(svc.updated_at) do
       true -> Preview.sync_service(svc)
       false -> :ok
     end

lib/console/deployments/services.ex

@@ -395,7 +395,7 @@ defmodule Console.Deployments.Services do
     start_transaction()
     |> add_operation(:source, fn _ ->
       get_service!(service_id)
-      |> Repo.preload([:dependencies, :context_bindings])
+      |> Repo.preload([:dependencies, :context_bindings, :imports])
       |> allow(user, :write)
     end)
     |> add_operation(:config, fn %{source: source} ->
@@ -407,9 +407,10 @@ defmodule Console.Deployments.Services do
       |> Console.dedupe(:git, Console.mapify(source.git))
       |> Console.dedupe(:helm, Console.mapify(source.helm))
       |> Console.dedupe(:kustomize, Console.mapify(source.kustomize))
+      |> Console.dedupe(:imports, Enum.map(source.imports, & %{service_id: &1.service_id, stack_id: &1.stack_id}))
       |> Console.dedupe(:dependencies, Enum.map(source.dependencies, & %{name: &1.name}))
       |> Console.dedupe(:sync_config, Console.clean(source.sync_config))
-      |> Map.merge(attrs)
+      |> DeepMerge.deep_merge(attrs)
       |> Map.put(:configuration, config)
       |> create_service(cluster_id, user)
     end)

lib/console/schema/pull_request.ex

@@ -87,6 +87,7 @@ defmodule Console.Schema.PullRequest do
     flow_id
     creator
     labels
+    preview
   )a
 
   def changeset(model, attrs \\ %{}) do

lib/oci/client.ex

@@ -28,13 +28,13 @@ defmodule Console.OCI.Client do
     put_in(client.client.options.dkr_repo, "#{repo}/#{suffix}")
   end
 
-  def tags(client, query \\ "", acc \\ %Tags{}) do
+  def tags(client, filter \\ fn _ -> true end, query \\ "", acc \\ %Tags{}) do
     case authed_get(client, "/v2/:repo/tags/list?n=1000#{query}") do
       {:ok, %Req.Response{status: 200, body: body, headers: %{"link" => _}}} ->
-        new = Tags.new(body)
-        tags(client, "&last=#{List.last(new.tags)}", merge_tags(acc, new))
+        new = Tags.new(body, filter)
+        tags(client, filter, "&last=#{List.last(new.tags)}", merge_tags(acc, new))
       {:ok, %Req.Response{status: 200, body: body}} ->
-        {:ok, merge_tags(acc, Tags.new(body))}
+        {:ok, merge_tags(acc, Tags.new(body, filter))}
       err -> handle_error(err)
     end
   end

lib/oci/schema.ex

@@ -52,6 +52,13 @@ defmodule Console.OCI.Tags do
   @type t :: %__MODULE__{}
   defstruct [:name, tags: []]
 
+  def new(body, filter) when is_function(filter, 1) do
+    %__MODULE__{
+      tags: Enum.filter(body["tags"], filter),
+      name: body["name"]
+    }
+  end
+
   def new(%{"tags" => tags, "name" => name}),
     do: %__MODULE__{tags: tags, name: name}
 

mix.exs

@@ -160,7 +160,7 @@ defmodule Console.MixProject do
       {:briefly, "~> 0.5.0"},
       {:libring, "~> 1.7"},
       {:http_stream, "~> 1.0.0"},
-      {:solid, "~> 0.15"},
+      {:solid, "~> 1.0.0-rc.0"},
       {:x509, "~> 0.8.5"},
       {:bamboo_phoenix, "~> 1.0"},
       {:slipstream, "~> 1.0"},

mix.lock

@@ -123,7 +123,7 @@
   "slipstream": {:hex, :slipstream, "1.2.0", "f4e820752339d86d2f15f4103e163f538e3bb9c7fe353225afd1255178e1f051", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mint_web_socket, "~> 0.2 or ~> 1.0", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.1 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f2fceddbb3c97331d348586e77c6425f4d150242dfaf392d22e8bd22f93d1f1e"},
   "snap": {:hex, :snap, "0.11.3", "12488b227929d340254bd3dc198f4819b134a361a57c367c1dce6a06ef192005", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3b80caf4f1bb11a505fee0c646c13687e4d87fda36a39b3bdc879265dabb8d0e"},
   "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"},
-  "solid": {:hex, :solid, "0.15.2", "6921af98a3a862041bb6af72b5f6e094dbf0242366b142f98a92cabe4ed30d2a", [:mix], [{:nimble_parsec, "~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "18b062b04948b7f7b99ac4a9360681dac7e0bd142df5e62a7761696c7384be45"},
+  "solid": {:hex, :solid, "1.0.0-rc.0", "57af6fd1c3defadad1a951ca56a27f612f290b2965a3a5f7b5a8cfe6012b11bd", [:mix], [], "hexpm", "135250f7f1fb849ac9554110628afea40cc41e4c457e670fe40cb22247190d99"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},

priv/pr/preview_env.md.eex

@@ -1,12 +1,16 @@
-Plural has deployed a preview environment for this PR.
+### <%= @status_emoji %> Deploy preview for <%= @service.name %> is <%= @status_text %>
 
-Service Name: <%= @service.name %>
-Service Status: <%= @service.component_status %> ![](https://img.shields.io/badge/<%= @service.status %>-<%= @status_color  %>)
-<%= if @comment_template do %>
-
-<details>
-<summary>Preview Environment details</summary>
-<%=  @comment_template %>
-</details>
+| Name  | Details |
+| :---: | ------- |
+| :zap: Service | <%= @service.name %> (<%= @service.component_status %> ready) |
+| :cloud: Cluster | <%= @service.cluster.name %> (<%= @service.cluster.distro %>) |
+| :octocat: Commit sha | <%= @sha %> |
+| ![](https://docs.plural.sh/favicon-16.png) Plural URL | [<%= Solid.StandardFilter.truncate(@url, 75) %>](<%= @url %>) |
+| :mag: Logs URL | [<%= Solid.StandardFilter.truncate(@logs_url, 75) %>](<%= @logs_url %>) |
+| :ocean: Flow URL | [<%= Solid.StandardFilter.truncate(@flow_url, 75) %>](<%= @flow_url %>) |<%= if @comment_template do %>
+<%= @comment_template %>
 <% end %>
-View your preview environment in the Plural Console [here](<%= @url %>)
+
+---
+
+You can learn more about preview environments and Plural Flows [here](https://docs.plural.sh/plural-features/flows)