Introducing additional technical indexes to Postgres to support nullable columns in unique indexes the same way Oracle does

morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java

@@ -15,6 +15,8 @@
 
 package org.alfasoftware.morf.jdbc;
 
+import static org.alfasoftware.morf.util.SchemaValidatorUtil.validateSchemaName;
+
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
@@ -54,8 +56,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 
-import static org.alfasoftware.morf.util.SchemaValidatorUtil.validateSchemaName;
-
 /**
  * Provides meta data based on a database connection.
  *
@@ -92,6 +92,11 @@ public class DatabaseMetaDataProvider implements Schema {
   protected static final int PRIMARY_COLUMN_NAME = 4;
   protected static final int PRIMARY_KEY_SEQ = 5;
 
+  // Keys for Database Information map
+  public static final String DATABASE_PRODUCT_VERSION = "DatabaseProductVersion";
+  public static final String DATABASE_MAJOR_VERSION = "DatabaseMajorVersion";
+  public static final String DATABASE_MINOR_VERSION = "DatabaseMinorVersion";
+
 
   protected final Connection connection;
   protected final String schemaName;
@@ -105,6 +110,8 @@ public class DatabaseMetaDataProvider implements Schema {
   private final Supplier<Map<AName, RealName>> viewNames = Suppliers.memoize(this::loadAllViewNames);
   private final LoadingCache<AName, View> viewCache = CacheBuilder.newBuilder().build(CacheLoader.from(this::loadView));
 
+  private final Supplier<Map<String, String>> databaseInformation = Suppliers.memoize(this::loadDatabaseInformation);
+
 
   /**
    * @param connection The database connection from which meta data should be provided.
@@ -117,6 +124,27 @@ protected DatabaseMetaDataProvider(Connection connection, String schemaName) {
   }
 
 
+  public Map<String, String> getDatabaseInformation() {
+    return databaseInformation.get();
+  }
+
+
+  private Map<String, String> loadDatabaseInformation() {
+    try {
+      final DatabaseMetaData databaseMetaData = connection.getMetaData();
+
+      return ImmutableMap.<String, String>builder()
+          .put(DATABASE_PRODUCT_VERSION, databaseMetaData.getDatabaseProductVersion())
+          .put(DATABASE_MAJOR_VERSION, String.valueOf(databaseMetaData.getDatabaseMajorVersion()))
+          .put(DATABASE_MINOR_VERSION, String.valueOf(databaseMetaData.getDatabaseMinorVersion()))
+          .build();
+    }
+    catch (SQLException e) {
+      throw new RuntimeSqlException(e);
+    }
+  }
+
+
   /**
    * @see org.alfasoftware.morf.metadata.Schema#isEmptyDatabase()
    */
@@ -693,7 +721,7 @@ protected List<Index> loadTableIndexes(RealName tableName) {
               continue;
             }
             if (DatabaseMetaDataProviderUtils.shouldIgnoreIndex(indexName.getDbName())) {
-              log.info("Ignoring index: ["+indexName.getDbName()+"]");
+              ignoreIndexName(indexName, indexResultSet);
               continue;
             }
 
@@ -714,6 +742,9 @@ protected List<Index> loadTableIndexes(RealName tableName) {
           catch (SQLException e) {
             throw new RuntimeSqlException("Error reading metadata for index ["+indexName+"] on table ["+tableName+"]", e);
           }
+          catch (RuntimeException e) {
+            throw new RuntimeException("Error reading metadata for index ["+indexName+"] on table ["+tableName+"]", e);
+          }
         }
 
         return indexColumns.entrySet().stream()
@@ -740,6 +771,21 @@ protected RealName readIndexName(ResultSet indexResultSet) throws SQLException {
   }
 
 
+  /**
+   * Receives an ignored index name.
+   * This can be overridden to process technical indexes, or remember performance indexes, etc.
+   *
+   * Note: This can be called multiple times for the same index!
+   *
+   * @param indexName name of the ignored index.
+   * @param indexResultSet resultset on the row with further index information
+   * @throws SQLException Upon errors.
+   */
+  protected void ignoreIndexName(RealName indexName, ResultSet indexResultSet) throws SQLException {
+    if (log.isDebugEnabled()) log.debug("Ignored index [" + indexName.getRealName() + "] column [" + indexResultSet.getString(INDEX_COLUMN_NAME) + "]");
+  }
+
+
   /**
    * Identify whether this is the primary key for this table.
    *
@@ -942,7 +988,7 @@ protected AName(String aName) {
       this.hashCode = aName.toLowerCase().hashCode();
     }
 
-    protected String getAName() {
+    public String getAName() {
       return aName;
     }
 

morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProviderUtils.java

@@ -36,6 +36,10 @@ public class DatabaseMetaDataProviderUtils {
    * Regex for extracting the data type from column comments
    */
   private static final Pattern DATA_TYPE_REGEX = Pattern.compile("TYPE:\\[(\\w+)\\]");
+  /**
+   * Regex for matching ignored indexes
+   */
+  private static final Pattern IGNORE_INDEX_REGEX = Pattern.compile(".*_PRF\\d+$");
 
   /**
    * Get the auto increment start value (if available) from the column comments
@@ -72,10 +76,14 @@ public static Optional<String> getDataTypeFromColumnComment(String columnComment
    *
    * eg. Schedule_PRF1
    *
+   * Also any indexes with a $ character will be ignored:
+   * this allows technical indexes to be added to the schema.
+   *
    * @param indexName The name of an index
    * @return Whether it should be ignored
    */
   public static boolean shouldIgnoreIndex(String indexName) {
-    return indexName.toUpperCase().matches(".*_PRF\\d+$");
+    return IGNORE_INDEX_REGEX.matcher(indexName.toUpperCase()).matches()
+        || indexName.indexOf('$') > -1;
   }
 }

morf-core/src/main/java/org/alfasoftware/morf/jdbc/SchemaResourceImpl.java

@@ -17,6 +17,7 @@
 
 import java.sql.Connection;
 import java.sql.SQLException;
+import java.util.Optional;
 
 import javax.sql.DataSource;
 
@@ -78,4 +79,15 @@ public void close() {
       throw new RuntimeSqlException("Closing", e);
     }
   }
+
+
+  @Override
+  public Optional<DatabaseMetaDataProvider> getDatabaseMetaDataProvider() {
+    if (delegate instanceof DatabaseMetaDataProvider) {
+      DatabaseMetaDataProvider metaDataProvider = (DatabaseMetaDataProvider)delegate;
+      return Optional.of(metaDataProvider);
+    }
+
+    return SchemaResource.super.getDatabaseMetaDataProvider();
+  }
 }

morf-core/src/main/java/org/alfasoftware/morf/jdbc/SqlDialect.java

@@ -48,6 +48,7 @@
 import org.alfasoftware.morf.metadata.DataValueLookup;
 import org.alfasoftware.morf.metadata.Index;
 import org.alfasoftware.morf.metadata.Schema;
+import org.alfasoftware.morf.metadata.SchemaResource;
 import org.alfasoftware.morf.metadata.SchemaUtils;
 import org.alfasoftware.morf.metadata.Table;
 import org.alfasoftware.morf.metadata.View;
@@ -4343,6 +4344,17 @@ protected String getSqlForInsertInto(@SuppressWarnings("unused") InsertStatement
 
 
 
+  /**
+   * Returns any statements needed to automatically heal the given schema.
+   *
+   * @param schemaResource Schema resource that can be examined.
+   * @return List of statements to be run.
+   */
+  public List<String> getSchemaConsistencyStatements(@SuppressWarnings("unused") SchemaResource schemaResource) {
+    return ImmutableList.of();
+  }
+
+
   /**
    * Class representing the structor of an ID Table.
    *

morf-core/src/main/java/org/alfasoftware/morf/metadata/SchemaResource.java

@@ -15,6 +15,10 @@
 
 package org.alfasoftware.morf.metadata;
 
+import java.util.Optional;
+
+import org.alfasoftware.morf.jdbc.DatabaseMetaDataProvider;
+
 /**
  * Provides database meta data.
  *
@@ -35,4 +39,13 @@ public interface SchemaResource extends Schema, AutoCloseable {
   @Override
   void close();
 
+
+  /**
+   * Returns the underlying {@link DatabaseMetaDataProvider}, if there is one.
+   *
+   * @return the underlying {@link DatabaseMetaDataProvider}, if there is one.
+   */
+  default Optional<DatabaseMetaDataProvider> getDatabaseMetaDataProvider() {
+    return Optional.empty();
+  }
 }

morf-core/src/main/java/org/alfasoftware/morf/upgrade/Upgrade.java

@@ -15,16 +15,20 @@
 
 package org.alfasoftware.morf.upgrade;
 
+import static org.alfasoftware.morf.metadata.SchemaUtils.copy;
 import static org.alfasoftware.morf.sql.SelectStatement.select;
 import static org.alfasoftware.morf.sql.SqlUtils.tableRef;
 import static org.alfasoftware.morf.sql.element.Function.count;
 import static org.alfasoftware.morf.upgrade.UpgradeStatus.NONE;
 
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -36,6 +40,7 @@
 import org.alfasoftware.morf.jdbc.SqlScriptExecutor.ResultSetProcessor;
 import org.alfasoftware.morf.jdbc.SqlScriptExecutorProvider;
 import org.alfasoftware.morf.metadata.Schema;
+import org.alfasoftware.morf.metadata.SchemaResource;
 import org.alfasoftware.morf.metadata.SchemaValidator;
 import org.alfasoftware.morf.sql.SelectStatement;
 import org.alfasoftware.morf.sql.element.TableReference;
@@ -45,15 +50,11 @@
 import org.alfasoftware.morf.upgrade.UpgradePath.UpgradePathFactoryImpl;
 import org.alfasoftware.morf.upgrade.UpgradePathFinder.NoUpgradePathExistsException;
 import org.alfasoftware.morf.upgrade.db.DatabaseUpgradeTableContribution;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
 
 
 /**
@@ -187,11 +188,21 @@ public UpgradePath findPath(Schema targetSchema, Collection<Class<? extends Upgr
 
     // Get access to the schema we are starting from
     log.info("Reading current schema");
-    Schema sourceSchema = UpgradeHelper.copySourceSchema(connectionResources, dataSource, exceptionRegexes);
-    SqlDialect dialect = connectionResources.sqlDialect();
+    List<String> schemaConsistencyStatements;
+    Schema sourceSchema;
+    try (SchemaResource databaseSchemaResource = connectionResources.openSchemaResource(dataSource)) {
+      sourceSchema = copy(databaseSchemaResource, exceptionRegexes);
+      schemaConsistencyStatements = connectionResources.sqlDialect().getSchemaConsistencyStatements(databaseSchemaResource);
+      if (!schemaConsistencyStatements.isEmpty()) {
+        log.warn("Auto-healing statements have been generated (" + schemaConsistencyStatements.size() + " statements in total); this usually implies auto-healing being carried out."
+            + " It this is shown on each subsequent start-up, it can be a symptom of auto-healing failing to achieve an acceptable stable healthful state."
+            + " Examine the upgrade statements (the upgrade script, or the upgrade logs below) to investigate further.");
+      }
+    }
 
     // -- Get the current UUIDs and deployed views...
     log.info("Examining current views");    //
+    SqlDialect dialect = connectionResources.sqlDialect();
     ExistingViewStateLoader existingViewState = new ExistingViewStateLoader(dialect, new ExistingViewHashLoader(dataSource, dialect), viewDeploymentValidator);
     Result viewChangeInfo = existingViewState.viewChanges(sourceSchema, targetSchema);
     ViewChanges viewChanges = new ViewChanges(targetSchema.views(), viewChangeInfo.getViewsToDrop(), viewChangeInfo.getViewsToDeploy());