feat: adaptation de l'etl pour une utilisation avec Agents en intervention et Scaleway

Author: sylvainlg

.env.example

@@ -4,3 +4,6 @@ METABASE_USERNAME=rdv_metabase
 RDV_SOLIDARITES_DB_URL=postgresql://user:pwd@localhost:10000/rdv_solidarites
 RDV_SERVICE_PUBLIC_DB_URL=postgresql://user:pwd@localhost:10000/rdv_service_public
 RDV_INSERTION_DB_URL=postgresql://user:pwd@localhost:10000/rdv_insertion
+
+# CONFIG_PATH=
+# ORIGIN_DB_URL=

Dockerfile

@@ -0,0 +1,31 @@
+FROM ruby:3.3.3-slim
+
+# Add PostgreSQL repository for version 16
+RUN apt-get update && apt-get install -y gnupg2 lsb-release wget \
+&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+&& echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+# Install PostgreSQL 16 client and other dependencies
+&& apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    libpq-dev \
+    postgresql-client-16 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy the Gemfile and Gemfile.lock
+COPY Gemfile Gemfile.lock ./
+
+# Copy the rest of the application
+COPY . .
+
+# Install gems
+RUN bundle install --jobs 4
+
+# Set environment variables
+ENV LANG=C.UTF-8
+
+# Command to run
+CMD ["echo", "use -it to run the container"]

Gemfile

@@ -9,4 +9,3 @@ gem "pg", "~> 1.5"
 gem "dotenv", "~> 3.1"
 
 gem "anonymizer", git: "https://github.com/betagouv/rdv-service-public.git", branch: "production", glob: "lib/anonymizer/anonymizer.gemspec"
-# gem "anonymizer", path: "../rdv-service-public/lib/anonymizer"

README.md

@@ -37,20 +37,26 @@ Le user Postgres de la base remplie par l’ETL utilisé par Metabase n’a pas
 
 ## Usage en staging et production
 
+### Hébergeur Scalingo
+
 ```shell
 scalingo --region osc-secnum-fr1 --app rdv-service-public-etl-staging run --detached "bundle exec ruby main.rb --app rdvi"
 ```
 
 Des CRON jobs réguliers seront bientôt configurés pour lancer ça.
 
+### Hébergeur Scaleway
+
+TODO
+
 ## Usage en local
 
 > [!WARNING]
 > Assurez vous de supprimer les fichiers de dumps, les bases de données restaurées, et les mots de passe des users Postgres utilisés en local après vos tests
 
 ### Variables d’environnement
 
-Copiez les variables d’environnement dans un fichier `.env` :
+Copiez le fichier `.env.example` puis renseignez les variables d’environnement dans un fichier `.env` :
 
 `cp .env.example .env`
 
@@ -99,3 +105,21 @@ Dans un autre terminal, lancer l’ETL :
 ```shell
 bundle exec ruby main.rb --app rdvs
 ```
+
+## Usage en local avec Docker
+
+Commencer par renseigner les variables d'environnement nécessaires dans le fichier `.env`, par exemple :
+
+```dotenv
+ETL_DB_URL=postgresql://esd:[password]@192.168.120.44:5432/aei-etl
+METABASE_USERNAME=esd
+ORIGIN_DB_URL=postgresql://esd:[password]@192.168.120.44:5432/api-aei
+CONFIG_PATH=https://gitlab.com/incubateur-territoires/startups/agents-intervention/agents-en-intervention/-/raw/feat/metabase/config.etl.yml
+```
+
+Puis :
+
+```shell
+docker build -t etl .
+docker run -it docker run --env-file .env -it etl bundle exec ruby main.rb --app [app]
+```

lib/etl.rb

@@ -8,16 +8,13 @@
 class Etl
   include Utils
 
-  VALID_APPS = %w[rdvi rdvs rdvsp].freeze
+  attr_reader :app, :etl_db_url, :origin_db_url, :config_path, :metabase_username
 
-  attr_reader :app, :etl_db_url, :rdv_db_url, :config_path, :metabase_username
-
-  def initialize(app:, etl_db_url:, rdv_db_url:, config_path:, metabase_username:)
+  def initialize(app:, etl_db_url:, origin_db_url:, config_path:, metabase_username:)
     @app = app
-    raise 'invalid app' if VALID_APPS.exclude?(app)
 
     @etl_db_url = etl_db_url
-    @rdv_db_url = rdv_db_url
+    @origin_db_url = origin_db_url
     @config_path = config_path
     @metabase_username = metabase_username
   end
@@ -30,8 +27,8 @@ def run
     @config = Anonymizer::Config.new(YAML.safe_load(File.read(config_path)))
 
     # make sure RDV db connection works
-    log_around "connect to RDV database #{rdv_db_url}" do
-      ActiveRecord::Base.establish_connection rdv_db_url
+    log_around "connect to origin database #{origin_db_url}" do
+      ActiveRecord::Base.establish_connection origin_db_url
       ActiveRecord::Base.connection # triggers connection
     end
 
@@ -54,7 +51,7 @@ def run
           pg_dump --clean --no-privileges --format tar \
             #{@config.truncated_table_names.map { "--exclude-table #{_1}" }.join(' ')} \
             -f #{dump_filename} \
-            #{rdv_db_url}
+            #{origin_db_url}
         SH
       )
     end
@@ -84,8 +81,13 @@ def run
     # workaround for a problematic column that we could also exclude
     # ERROR:  cannot insert a non-DEFAULT value into column "text_search_terms" (PG::GeneratedAlways)
     # DÉTAIL : Column "text_search_terms" and "text_search_terms_with_notification_email" are generated columns.
-    run_sql_command %(ALTER TABLE users DROP COLUMN IF EXISTS text_search_terms CASCADE)
-    run_sql_command %(ALTER TABLE users DROP COLUMN IF EXISTS text_search_terms_with_notification_email CASCADE)
+    # Vérifier que la table users existe avant de faire l'alter table
+    if ActiveRecord::Base.connection.table_exists?("users")
+      run_sql_command %(ALTER TABLE users DROP COLUMN IF EXISTS text_search_terms CASCADE)
+      run_sql_command %(ALTER TABLE users DROP COLUMN IF EXISTS text_search_terms_with_notification_email CASCADE)
+    else
+      logger.info "La table users n'existe pas, aucune modification n'est appliquée."
+    end
 
     # STEP : move from public to target schema
     target_schema = app

main.rb

@@ -11,40 +11,29 @@
 
 app = nil
 OptionParser.new do |opts|
-  opts.on('-a', '--app APP', Etl::VALID_APPS) { app = _1 }
+  opts.on('-a', '--app APP') { app = _1 }
 end.parse!
 
-config_path = if ENV["CONFIG_PATH"] && File.exist?(ENV["CONFIG_PATH"])
-  ENV["CONFIG_PATH"]
-else
-  config_url = {
-    "rdvi" => "https://raw.githubusercontent.com/betagouv/rdv-insertion/main/config/anonymizer.yml",
-    "rdvs" => "https://raw.githubusercontent.com/betagouv/rdv-service-public/production/config/anonymizer.yml",
-    "rdvsp" => "https://raw.githubusercontent.com/betagouv/rdv-service-public/production/config/anonymizer.yml"
-  }[app]
-  run_command "curl -o config.yml \"#{config_url}\""
-  "config.yml"
+unless ENV["CONFIG_PATH"] && File.exist?(ENV["CONFIG_PATH"])
+  raise "La variable d'environnement CONFIG_PATH n'est pas définie ou pointe vers un fichier inexistant"
 end
 
-rdv_db_url_env_var = {
-  "rdvi" => "RDV_INSERTION_DB_URL",
-  "rdvs" => "RDV_SOLIDARITES_DB_URL",
-  "rdvsp" => "RDV_SERVICE_PUBLIC_DB_URL"
-}[app]
+config_path = ENV["CONFIG_PATH"]
 
+origin_db_url_env_var = "ORIGIN_DB_URL"
 etl_db_url_env_var = "ETL_DB_URL"
 metabase_username_env_var = "METABASE_USERNAME"
 
 [
-  rdv_db_url_env_var,
+  origin_db_url_env_var,
   etl_db_url_env_var,
   metabase_username_env_var
 ].each do |env_var|
   raise "Missing environment variable #{env_var}" if ENV[env_var].blank?
 end
 
-rdv_db_url = ENV[rdv_db_url_env_var]
+origin_db_url = ENV[origin_db_url_env_var]
 etl_db_url = ENV[etl_db_url_env_var]
 metabase_username = ENV[metabase_username_env_var]
 
-Etl.new(app:, etl_db_url:, rdv_db_url:, config_path:, metabase_username:).run
+Etl.new(app:, etl_db_url:, origin_db_url:, config_path:, metabase_username:).run

tests/test_etl.rb

@@ -19,7 +19,7 @@ def test_something
     Etl.new(
       app: "rdvs",
       etl_db_url: "postgresql://localhost/rdv_sp_etl_test_target",
-      rdv_db_url: "postgresql://localhost/rdv_sp_etl_test_source",
+      origin_db_url: "postgresql://localhost/rdv_sp_etl_test_source",
       config_path: File.expand_path("config.yml", File.dirname(__FILE__)),
       metabase_username: "rdv_sp_etl_metabase_user"
     ).run