helpers for getting resources lists from database

Author: vdegove

apps/transport/lib/db/irve_valid_file.ex

@@ -5,6 +5,7 @@ defmodule DB.IRVEValidFile do
   """
   use Ecto.Schema
   use TypedEctoSchema
+  import Ecto.Query
 
   typed_schema "irve_valid_file" do
     field(:datagouv_dataset_id, :string, null: false)
@@ -16,4 +17,36 @@ defmodule DB.IRVEValidFile do
     has_many(:irve_valid_pdcs, DB.IRVEValidPDC, foreign_key: :irve_valid_file_id)
     timestamps(type: :utc_datetime_usec)
   end
+
+  @doc """
+  Set of `datagouv_resource_id`s currently present in the database. Meant to be snapshotted
+  before a consolidation run, to tell whether a datagouv resource already had a version imported.
+  """
+  def existing_datagouv_resource_ids do
+    DB.IRVEValidFile
+    |> distinct(true)
+    |> select([f], f.datagouv_resource_id)
+    |> DB.Repo.all()
+    |> MapSet.new()
+  end
+
+  @doc """
+  File-level metadata and stored PDC count, for the given (typically orphan) `datagouv_resource_id`s.
+  Used to report resources still in the database but no longer listed on data.gouv.fr.
+  """
+  def orphan_files(datagouv_resource_ids) do
+    DB.IRVEValidFile
+    |> where([f], f.datagouv_resource_id in ^datagouv_resource_ids)
+    |> join(:left, [f], p in DB.IRVEValidPDC, on: p.irve_valid_file_id == f.id)
+    |> group_by([f], f.id)
+    |> select([f, p], %{
+      datagouv_dataset_id: f.datagouv_dataset_id,
+      datagouv_resource_id: f.datagouv_resource_id,
+      dataset_title: f.dataset_title,
+      datagouv_organization_or_owner: f.datagouv_organization_or_owner,
+      datagouv_last_modified: f.datagouv_last_modified,
+      pdc_count: count(p.id)
+    })
+    |> DB.Repo.all()
+  end
 end

apps/transport/test/db/irve_valid_file_test.exs

@@ -0,0 +1,66 @@
+defmodule DB.IRVEValidFileTest do
+  use ExUnit.Case, async: true
+
+  setup do
+    :ok = Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo)
+  end
+
+  defp insert_file!(resource_id, attrs \\ %{}) do
+    DB.Repo.insert!(%DB.IRVEValidFile{
+      datagouv_dataset_id: Map.get(attrs, :dataset_id, "dataset-#{resource_id}"),
+      datagouv_resource_id: resource_id,
+      checksum: Map.get(attrs, :checksum, "checksum-#{resource_id}"),
+      dataset_title: Map.get(attrs, :dataset_title, "title-#{resource_id}"),
+      datagouv_organization_or_owner: Map.get(attrs, :org, "org-#{resource_id}")
+    })
+  end
+
+  describe "existing_datagouv_resource_ids/0" do
+    test "returns the distinct set of datagouv_resource_ids" do
+      insert_file!("resource-a")
+      insert_file!("resource-b")
+      # same resource, another version: must be deduplicated
+      insert_file!("resource-a", %{checksum: "checksum-a-2"})
+
+      assert DB.IRVEValidFile.existing_datagouv_resource_ids() ==
+               MapSet.new(["resource-a", "resource-b"])
+    end
+  end
+
+  describe "orphan_files/1" do
+    test "returns metadata and PDC count for the given resource ids only" do
+      path = System.tmp_dir!() |> Path.join("irve_orphan_#{Ecto.UUID.generate()}.csv")
+
+      File.write!(
+        path,
+        DB.Factory.IRVE.to_csv_body([
+          DB.Factory.IRVE.generate_row(%{"id_pdc_itinerance" => "FRPAN99E00000001"}),
+          DB.Factory.IRVE.generate_row(%{"id_pdc_itinerance" => "FRPAN99E00000002"})
+        ])
+      )
+
+      Transport.IRVE.DatabaseImporter.write_to_db(
+        path,
+        "orphan-dataset-id",
+        "orphan-resource-id",
+        "orphan-dataset-title",
+        "orphan-org",
+        "2024-01-01T10:00:00+00:00"
+      )
+
+      insert_file!("kept-resource-id")
+
+      assert [orphan] = DB.IRVEValidFile.orphan_files(["orphan-resource-id"])
+
+      assert %{
+               datagouv_dataset_id: "orphan-dataset-id",
+               datagouv_resource_id: "orphan-resource-id",
+               dataset_title: "orphan-dataset-title",
+               datagouv_organization_or_owner: "orphan-org",
+               pdc_count: 2
+             } = orphan
+
+      assert DB.IRVEValidFile.orphan_files([]) == []
+    end
+  end
+end