Add splitting of long SQL queries in DBQLSqlTbl (#312)

The `SqlTextInfo` column of the `dbc.DBQLSqlTbl` table may need
splitting due to 64kB row size limit on Teradata 15. This change
introduces a new command-line option that allows for splitting
this column into multiple rows if the length is longer than
the specified max length.

By default, if the option is not used the column is extracted as is,
which may reach 31000 characters. With the option, the column is
split into multiple rows. The max length can be any integer
between 5000 and 30000 characters.

Example usage:

```
-Dteradata-logs.max-sql-length=7000
```

Author: dawidxc

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java

@@ -21,6 +21,7 @@
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
 import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
 import java.util.List;
+import java.util.OptionalLong;
 import java.util.function.Predicate;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
@@ -162,8 +163,18 @@ public TeradataAssessmentLogsJdbcTask(
       List<String> conditions,
       ZonedInterval interval,
       @CheckForNull String logDateColumn,
+      OptionalLong maxSqlLength,
       List<String> orderBy) {
-    super(targetPath, state, logTable, queryTable, conditions, interval, logDateColumn, orderBy);
+    super(
+        targetPath,
+        state,
+        logTable,
+        queryTable,
+        conditions,
+        interval,
+        logDateColumn,
+        maxSqlLength,
+        orderBy);
   }
 
   @Nonnull

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java

@@ -21,6 +21,7 @@
 import com.google.auto.service.AutoService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Range;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment;
@@ -38,13 +39,15 @@
 import com.google.edwmigration.dumper.application.dumper.task.FormatTask;
 import com.google.edwmigration.dumper.application.dumper.task.Task;
 import com.google.edwmigration.dumper.application.dumper.task.TaskCategory;
+import com.google.edwmigration.dumper.application.dumper.utils.PropertyParser;
 import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description;
 import com.google.edwmigration.dumper.plugin.lib.dumper.spi.TeradataLogsDumpFormat;
 import java.time.Duration;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.OptionalLong;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -62,6 +65,10 @@ public class TeradataLogsConnector extends AbstractTeradataConnector
 
   private static final Logger LOG = LoggerFactory.getLogger(TeradataLogsConnector.class);
   private static final String ASSESSMENT_DEF_LOG_TABLE = "dbc.QryLogV";
+  static final int DBQLSQLTBL_SQLTEXTINFO_LENGTH = 31000;
+
+  private static final Range<Long> MAX_SQL_LENGTH_RANGE =
+      Range.closed(5000L, (long) DBQLSQLTBL_SQLTEXTINFO_LENGTH);
 
   public enum TeradataLogsConnectorProperty implements ConnectorProperty {
     UTILITY_LOGS_TABLE(
@@ -81,6 +88,12 @@ public enum TeradataLogsConnectorProperty implements ConnectorProperty {
         "The name of the column of type DATE to include in the WHERE clause when dumping"
             + " query log tables. The column must exist in both tables."
             + " See --query-log-alternates for query log table names.",
+        /* defaultValue= */ null),
+    MAX_SQL_LENGTH(
+        "max-sql-length",
+        "Max length of the DBQLSqlTbl.SqlTextInfo column."
+            + " Text that is longer than the defined limit will be split into multiple rows."
+            + " Example: 10000. Allowed range: 5000-30000.",
         /* defaultValue= */ null);
 
     private final String name;
@@ -174,9 +187,6 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
       conditions.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp());
     }
 
-    // Beware of Teradata SQLSTATE HY000. See issue #4126.
-    // Most likely caused by some operation (equality?) being performed on a datum which is too long
-    // for a varchar.
     Duration rotationDuration = arguments.getQueryLogRotationFrequency();
     ZonedIntervalIterable intervals =
         ZonedIntervalIterableGenerator.forConnectorArguments(
@@ -187,6 +197,9 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
     String utilityLogsTable =
         arguments.getDefinitionOrDefault(TeradataLogsConnectorProperty.UTILITY_LOGS_TABLE);
     String logDateColumn = arguments.getDefinition(TeradataLogsConnectorProperty.LOG_DATE_COLUMN);
+    OptionalLong maxSqlLength =
+        PropertyParser.parseNumber(
+            arguments, TeradataLogsConnectorProperty.MAX_SQL_LENGTH, MAX_SQL_LENGTH_RANGE);
     for (ZonedInterval interval : intervals) {
       String file = createFilename(ZIP_ENTRY_PREFIX, interval);
       if (isAssessment) {
@@ -200,6 +213,7 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
                     conditions,
                     interval,
                     logDateColumn,
+                    maxSqlLength,
                     orderBy)
                 .withHeaderClass(HeaderForAssessment.class));
         out.addAll(createTimeSeriesTasks(interval, arguments));

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java

@@ -16,13 +16,17 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.teradata;
 
+import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataLogsConnector.DBQLSQLTBL_SQLTEXTINFO_LENGTH;
 import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.formatQuery;
+import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.optionalIf;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
 import com.google.common.io.ByteSink;
+import com.google.common.primitives.Ints;
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
 import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
 import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle;
@@ -36,6 +40,7 @@
 import java.time.format.DateTimeFormatter;
 import java.util.Collections;
 import java.util.List;
+import java.util.OptionalLong;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import javax.annotation.CheckForNull;
@@ -93,6 +98,7 @@ public class TeradataLogsJdbcTask extends AbstractJdbcTask<Summary> {
   protected final List<String> conditions;
   protected final ZonedInterval interval;
   @CheckForNull private final String logDateColumn;
+  private final OptionalLong maxSqlLength;
   protected final List<String> orderBy;
 
   public TeradataLogsJdbcTask(
@@ -110,6 +116,7 @@ public TeradataLogsJdbcTask(
         conditions,
         interval,
         /* logDateColumn= */ null,
+        /* maxSqlLength= */ OptionalLong.empty(),
         Collections.emptyList());
   }
 
@@ -121,6 +128,7 @@ protected TeradataLogsJdbcTask(
       List<String> conditions,
       ZonedInterval interval,
       @CheckForNull String logDateColumn,
+      OptionalLong maxSqlLength,
       List<String> orderBy) {
     super(targetPath);
     this.state = Preconditions.checkNotNull(state, "SharedState was null.");
@@ -129,6 +137,7 @@ protected TeradataLogsJdbcTask(
     this.conditions = conditions;
     this.interval = interval;
     this.logDateColumn = logDateColumn;
+    this.maxSqlLength = maxSqlLength;
     this.orderBy = orderBy;
   }
 
@@ -190,7 +199,16 @@ private String getSql(@Nonnull JdbcHandle handle) {
       // "QueryID is a system-wide unique field; you can use QueryID
       // to join DBQL tables ... without needing ProcID as an additional join field."
       // (https://docs.teradata.com/reader/B7Lgdw6r3719WUyiCSJcgw/YIKoBz~QQgv2Aw5dF339kA)
-      buf.append(" LEFT OUTER JOIN ").append(queryTable).append(" ST ON (L.QueryID=ST.QueryID");
+      buf.append(" LEFT OUTER JOIN ");
+      if (maxSqlLength.isPresent()) {
+        buf.append('(')
+            .append(
+                createSubQueryWithSplittingLongQueries(Ints.checkedCast(maxSqlLength.getAsLong())))
+            .append(')');
+      } else {
+        buf.append(queryTable);
+      }
+      buf.append(" ST ON (L.QueryID=ST.QueryID");
 
       if (logDateColumn != null) {
         buf.append(" AND L.").append(logDateColumn).append("=ST.").append(logDateColumn);
@@ -214,12 +232,7 @@ private String getSql(@Nonnull JdbcHandle handle) {
             SQL_FORMAT.format(interval.getStart()), SQL_FORMAT.format(interval.getEndExclusive())));
 
     if (logDateColumn != null) {
-      buf.append(" AND L.")
-          .append(logDateColumn)
-          .append(" = ")
-          .append("CAST('")
-          .append(SQL_DATE_FORMAT.format(interval.getStart()))
-          .append("' AS DATE)");
+      buf.append(" AND L.").append(createLogDateColumnCondition());
     }
 
     for (String condition : conditions) {
@@ -233,6 +246,27 @@ private String getSql(@Nonnull JdbcHandle handle) {
     return buf.toString().replace('\n', ' ');
   }
 
+  private String createLogDateColumnCondition() {
+    return logDateColumn + " = CAST('" + SQL_DATE_FORMAT.format(interval.getStart()) + "' AS DATE)";
+  }
+
+  private String createSubQueryWithSplittingLongQueries(int maxLength) {
+    ImmutableList.Builder<String> columns = ImmutableList.<String>builder().add("QueryID");
+    if (logDateColumn != null) {
+      columns.add(logDateColumn);
+    }
+    return new SplitTextColumnQueryGenerator(
+            columns.build(),
+            "SqlTextInfo",
+            "SqlRowNo",
+            queryTable,
+            /* whereCondition=*/ optionalIf(
+                logDateColumn != null, this::createLogDateColumnCondition),
+            DBQLSQLTBL_SQLTEXTINFO_LENGTH,
+            maxLength)
+        .generate();
+  }
+
   /**
    * Runs a test query to check whether a given projection expression is legal on this Teradata
    * instance.

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataMetadataConnector.java

@@ -17,6 +17,7 @@
 package com.google.edwmigration.dumper.application.dumper.connector.teradata;
 
 import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.formatQuery;
+import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.optionalIf;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 
@@ -42,7 +43,6 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.OptionalLong;
-import java.util.function.Supplier;
 import javax.annotation.Nonnull;
 
 /** @author miguel */
@@ -70,7 +70,7 @@ public enum TeradataMetadataConnectorProperties implements ConnectorProperty {
     MAX_TEXT_LENGTH(
         "max-text-length",
         "Max length of the text column when dumping TableTextV view."
-            + " Text that is longer than the defined limit will be split."
+            + " Text that is longer than the defined limit will be split into multiple rows."
             + " Example: 10000. Allowed range: "
             + MAX_TEXT_LENGTH_RANGE
             + ".");
@@ -263,10 +263,6 @@ private TeradataJdbcSelectTask createTaskForTableTextV(ConnectorArguments argume
         TableTextVFormat.ZIP_ENTRY_NAME, TaskCategory.REQUIRED, query);
   }
 
-  private static <T> Optional<T> optionalIf(boolean condition, Supplier<T> supplier) {
-    return condition ? Optional.of(supplier.get()) : Optional.empty();
-  }
-
   private static String escapeStringLiteral(String s) {
     return "'" + (s.replaceAll("'", "''")) + "'";
   }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataUtils.java

@@ -18,9 +18,14 @@
 
 import com.google.common.base.Preconditions;
 import java.util.Optional;
+import java.util.function.Supplier;
 
 class TeradataUtils {
 
+  public static <T> Optional<T> optionalIf(boolean condition, Supplier<T> supplier) {
+    return condition ? Optional.of(supplier.get()) : Optional.empty();
+  }
+
   /**
    * Formats the query by:
    *