concord-server-db: option to host process log objects in another DB

Author: ibodrov

server/db/pom.xml

@@ -16,7 +16,8 @@
     <properties>
         <db.image>library/postgres:10.4-alpine</db.image>
         <db.baseDir>${project.build.directory}/db</db.baseDir>
-        <db.changeLogPath>com/walmartlabs/concord/server/db/liquibase.xml</db.changeLogPath>
+        <mainDb.changeLogPath>com/walmartlabs/concord/server/db/mainDb.xml</mainDb.changeLogPath>
+        <logDb.changeLogPath>com/walmartlabs/concord/server/db/logDb.xml</logDb.changeLogPath>
         <db.host>localhost</db.host>
         <db.username>postgres</db.username>
         <db.password>q1</db.password>
@@ -164,14 +165,27 @@
                 <version>${liquibase.version}</version>
                 <executions>
                     <execution>
+                        <id>main-db</id>
                         <phase>generate-sources</phase>
                         <goals>
                             <goal>update</goal>
                         </goals>
+                        <configuration>
+                            <changeLogFile>src/main/resources/${mainDb.changeLogPath}</changeLogFile>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>log-db</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>update</goal>
+                        </goals>
+                        <configuration>
+                            <changeLogFile>src/main/resources/${logDb.changeLogPath}</changeLogFile>
+                        </configuration>
                     </execution>
                 </executions>
                 <configuration>
-                    <changeLogFile>src/main/resources/${db.changeLogPath}</changeLogFile>
                     <driver>org.postgresql.Driver</driver>
                     <url>jdbc:postgresql://${db.host}:${db.port}/postgres</url>
                     <username>${db.username}</username>

server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java

@@ -27,8 +27,10 @@
 
 import javax.inject.Singleton;
 import javax.sql.DataSource;
+import java.lang.annotation.Annotation;
 import java.util.Comparator;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.google.inject.multibindings.Multibinder.newSetBinder;
 
@@ -47,46 +49,83 @@ public DatabaseModule(boolean migrateDb) {
     @Override
     public void configure(Binder binder) {
         newSetBinder(binder, DatabaseChangeLogProvider.class).addBinding().to(MainDBChangeLogProvider.class);
+        newSetBinder(binder, DatabaseChangeLogProvider.class).addBinding().to(LogDBChangeLogProvider.class);
     }
 
     @Provides
     @MainDB
     @Singleton
-    public DataSource appDataSource(@MainDB DatabaseConfiguration cfg,
+    public DataSource mainDbDataSource(@MainDB DatabaseConfiguration cfg,
+                                       MetricRegistry metricRegistry,
+                                       Set<DatabaseChangeLogProvider> changeLogProviders) {
+
+        var ds = DataSourceUtils.createDataSource(cfg, "app" /* not called "main" for backward compatibility */, cfg.username(), cfg.password(), metricRegistry);
+        if (migrateDb) {
+            migrateDb(changeLogProviders, ds, cfg, MainDB.class);
+        }
+        return ds;
+    }
+
+    @Provides
+    @LogDB
+    @Singleton
+    public DataSource logDataSource(@LogDB DatabaseConfiguration cfg,
                                     MetricRegistry metricRegistry,
                                     Set<DatabaseChangeLogProvider> changeLogProviders) {
 
-        DataSource ds = DataSourceUtils.createDataSource(cfg, "app", cfg.username(), cfg.password(), metricRegistry);
-
+        var ds = DataSourceUtils.createDataSource(cfg, "log", cfg.username(), cfg.password(), metricRegistry);
         if (migrateDb) {
-            changeLogProviders.stream()
-                    // can't inject a set of objects with the same qualifier, filter manually
-                    .filter(p -> p.getClass().getAnnotation(MainDB.class) != null)
-                    .sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order))
-                    .forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters()));
+            migrateDb(changeLogProviders, ds, cfg, LogDB.class);
         }
-
         return ds;
     }
 
     @Provides
     @JsonStorageDB
     @Singleton
-    public DataSource inventoryDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) {
+    public DataSource jsonStorageDbDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) {
         return DataSourceUtils.createDataSource(cfg, "inventory", cfg.username(), cfg.password(), metricRegistry);
     }
 
     @Provides
     @MainDB
     @Singleton
-    public Configuration appJooqConfiguration(@MainDB DataSource ds) {
+    public Configuration mainDbJooqConfiguration(@MainDB DataSource ds) {
+        return DataSourceUtils.createJooqConfiguration(ds);
+    }
+
+    @Provides
+    @LogDB
+    @Singleton
+    public Configuration logDbJooqConfiguration(@LogDB DataSource ds) {
         return DataSourceUtils.createJooqConfiguration(ds);
     }
 
     @Provides
     @JsonStorageDB
     @Singleton
-    public Configuration inventoryJooqConfiguration(@JsonStorageDB DataSource ds) {
+    public Configuration jsonStorageDbJooqConfiguration(@JsonStorageDB DataSource ds) {
         return DataSourceUtils.createJooqConfiguration(ds);
     }
+
+    private static void migrateDb(Set<DatabaseChangeLogProvider> changeLogProviders,
+                                  DataSource ds,
+                                  DatabaseConfiguration cfg,
+                                  Class<? extends Annotation> annotation) {
+
+        var providers = changeLogProviders.stream()
+                // can't inject a set of objects with the same qualifier, filter manually
+                .filter(p -> p.getClass().getAnnotation(annotation) != null)
+                .sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order))
+                .toList();
+
+        if (providers.isEmpty()) {
+            // classpath issue or a bug?
+            var availableProviders = changeLogProviders.stream().map(DatabaseChangeLogProvider::getChangeLogPath).sorted();
+            throw new IllegalStateException("Can't find a DatabaseChangeLogProvider for %s (most likely a bug). Available providers: %s"
+                    .formatted(annotation.getName(), availableProviders.collect(Collectors.joining(", "))));
+        }
+
+        providers.forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters()));
+    }
 }

server/db/src/main/java/com/walmartlabs/concord/db/LogDB.java

@@ -0,0 +1,30 @@
+package com.walmartlabs.concord.db;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2019 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import javax.inject.Qualifier;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Qualifier
+public @interface LogDB {
+}

server/db/src/main/java/com/walmartlabs/concord/db/LogDBChangeLogProvider.java

@@ -0,0 +1,41 @@
+package com.walmartlabs.concord.db;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2024 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+@LogDB
+public class LogDBChangeLogProvider implements DatabaseChangeLogProvider {
+
+    @Override
+    public String getChangeLogPath() {
+        return "com/walmartlabs/concord/server/db/logDb.xml";
+    }
+
+    @Override
+    public int order() {
+        // we expect the log DB to be migrated after the main DB
+        return 1;
+    }
+
+    @Override
+    public String toString() {
+        return "log-db";
+    }
+}

server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java

@@ -25,17 +25,17 @@ public class MainDBChangeLogProvider implements DatabaseChangeLogProvider {
 
     @Override
     public String getChangeLogPath() {
-        return "com/walmartlabs/concord/server/db/liquibase.xml";
+        return "com/walmartlabs/concord/server/db/mainDb.xml";
     }
 
     @Override
     public int order() {
-        // we expect the server's DB to be migrated first
+        // we expect the main DB to be migrated first
         return 0;
     }
 
     @Override
     public String toString() {
-        return "server-db";
+        return "main-db";
     }
 }