Add --max-tablesizev-rows parameter. (#152)

* Add --max-tablesizev-rows parameter.

Add possibility to limit the number of rows returned from the DBC.TableSizeV table.

Author: dawidxc

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/ConnectorArguments.java

@@ -46,6 +46,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -97,6 +98,7 @@ public class ConnectorArguments extends DefaultArguments {
     public static final String OPT_DATABASE = "database";
     public static final String OPT_SCHEMA = "schema";
     public static final String OPT_ASSESSMENT = "assessment";
+    public static final String OPT_TERADATA_MAX_TABLESIZEV_ROWS = "max-tablesizev-rows";
     public static final String OPT_ORACLE_SID = "oracle-sid";
     public static final String OPT_ORACLE_SERVICE = "oracle-service";
 
@@ -131,6 +133,13 @@ public class ConnectorArguments extends DefaultArguments {
     private final OptionSpec<String> optionDatabase = parser.accepts(OPT_DATABASE, "Database(s) to export").withRequiredArg().ofType(String.class).withValuesSeparatedBy(',').describedAs("db0,db1,...");
     private final OptionSpec<String> optionSchema = parser.accepts(OPT_SCHEMA, "Schemata to export").withRequiredArg().ofType(String.class).withValuesSeparatedBy(',').describedAs("sch0,sch1,...");
     private final OptionSpec<Void> optionAssessment = parser.accepts(OPT_ASSESSMENT, "Whether to create a dump for assessment (i.e., dump additional information).");
+
+    public static final String TERADATA_MAX_TABLE_SIZE_V_ROWS_DESCRIPTION =
+        "Max number of rows to extract from DBC.TableSizeV table (available for 'teradata' connector only)";
+    private final OptionSpec<Long> optionTeradataMaxTableSizeVRows =
+        parser.accepts(OPT_TERADATA_MAX_TABLESIZEV_ROWS, TERADATA_MAX_TABLE_SIZE_V_ROWS_DESCRIPTION)
+        .withRequiredArg().ofType(Long.class).describedAs("100000");
+
     private final OptionSpec<String> optionUser = parser.accepts(OPT_USER, "Database username").withRequiredArg().describedAs("admin");
     private final OptionSpec<String> optionPass = parser.accepts(OPT_PASSWORD, "Database password, prompted if not provided").withOptionalArg().describedAs("sekr1t");
     private final OptionSpec<String> optionRole = parser.accepts(OPT_ROLE, "Database role").withRequiredArg().describedAs("dumper");
@@ -417,6 +426,13 @@ public boolean isAssessment() {
         return getOptions().has(optionAssessment);
     }
 
+    public Optional<Long> getTeradataMaxTableSizeVRows() {
+        if (!getOptions().has(optionTeradataMaxTableSizeVRows)) {
+            return Optional.empty();
+        }
+        return Optional.of(getOptions().valueOf(optionTeradataMaxTableSizeVRows));
+    }
+
     @Nonnull
     public Predicate<String> getSchemaPredicate() {
         return toPredicate(getSchemata());

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataMetadataConnector.java

@@ -18,6 +18,7 @@
 
 import com.google.auto.service.AutoService;
 import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment;
+import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput;
 import java.util.List;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
 import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabasePredicate;
@@ -41,6 +42,8 @@
 @Description("Dumps metadata from Teradata.")
 @RespectsArgumentDatabasePredicate
 @RespectsArgumentAssessment
+@RespectsInput(order = 450, arg = ConnectorArguments.OPT_TERADATA_MAX_TABLESIZEV_ROWS,
+    description = ConnectorArguments.TERADATA_MAX_TABLE_SIZE_V_ROWS_DESCRIPTION)
 public class TeradataMetadataConnector extends AbstractTeradataConnector implements MetadataConnector, TeradataMetadataDumpFormat {
 
     @SuppressWarnings("UnusedVariable")
@@ -113,14 +116,7 @@ public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
                     TaskCategory.OPTIONAL,
                     "SELECT %s FROM DBC.StatsV " + whereDatabaseNameClause + " ;"));
 
-          // TableSizeV contains a row per each VProc/AMP, so it can grow significantly for large dbs.
-          // Hence, we aggregate before dumping.
-          // See recommended usage
-          // https://docs.teradata.com/r/Teradata-VantageTM-Data-Dictionary/March-2019/Views-Reference/TableSizeV-X/Examples-Using-TableSizeV
-          out.add(new TeradataJdbcSelectTask(TableSizeVFormat.ZIP_ENTRY_NAME,
-              TaskCategory.OPTIONAL,
-              "SELECT DataBaseName, AccountName, TableName, SUM(CurrentPerm) CurrentPerm, SUM(PeakPerm) PeakPerm FROM DBC.TableSizeV "
-                  + whereDataBaseNameClause + " GROUP BY 1,2,3;"));
+            out.add(createTaskForTableSizeV(whereDataBaseNameClause, arguments));
 
             out.add(new TeradataJdbcSelectTask(AllTempTablesVXFormat.ZIP_ENTRY_NAME,
                 TaskCategory.OPTIONAL,
@@ -144,38 +140,25 @@ public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
         }
     }
 
-}
-
+    private TeradataJdbcSelectTask createTaskForTableSizeV(
+        String whereDataBaseNameClause,
+        ConnectorArguments arguments) {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT ");
+        arguments.getTeradataMaxTableSizeVRows().ifPresent(maxRows -> query.append("TOP ").append(maxRows));
+        // TableSizeV contains a row per each VProc/AMP, so it can grow significantly for large dbs.
+        // Hence, we aggregate before dumping.
+        // See recommended usage
+        // https://docs.teradata.com/r/Teradata-VantageTM-Data-Dictionary/March-2019/Views-Reference/TableSizeV-X/Examples-Using-TableSizeV
+        query.append(" DataBaseName, AccountName, TableName, SUM(CurrentPerm) CurrentPerm, SUM(PeakPerm) PeakPerm FROM DBC.TableSizeV ")
+            .append(whereDataBaseNameClause)
+            .append(" GROUP BY 1,2,3");
+        arguments.getTeradataMaxTableSizeVRows().ifPresent(unused -> query.append(" ORDER BY 4 DESC"));
+        query.append(';');
+        return new TeradataJdbcSelectTask(
+            TableSizeVFormat.ZIP_ENTRY_NAME,
+            TaskCategory.OPTIONAL,
+            query.toString());
+    }
 
-/*
-14:44:43.475 [main] WARN  com.google.edwmigration.dumper.application.dumper.MetadataDumper - Task failed: Write dbc.ColumnsqV.csv from:
-SELECT * FROM DBC.ColumnsqV ORDER BY 1,2,3,4;: org.springframework.jdbc.UncategorizedSQLException: StatementCallback; uncategorized SQLException for SQL [SELECT * FROM DBC.ColumnsqV ORDER BY 1,2,3,4;]; SQL state [HY000]; error code [9719]; [Teradata Database] [TeraJDBC 16.20.00.12] [Error 9719] [SQLState HY000] QVCI feature is disabled.; nested exception is java.sql.SQLException: [Teradata Database] [TeraJDBC 16.20.00.12] [Error 9719] [SQLState HY000] QVCI feature is disabled.
-org.springframework.jdbc.UncategorizedSQLException: StatementCallback; uncategorized SQLException for SQL [SELECT * FROM DBC.ColumnsqV ORDER BY 1,2,3,4;]; SQL state [HY000]; error code [9719]; [Teradata Database] [TeraJDBC 16.20.00.12] [Er
-or 9719] [SQLState HY000] QVCI feature is disabled.; nested exception is java.sql.SQLException: [Teradata Database] [TeraJDBC 16.20.00.12] [Error 9719] [SQLState HY000] QVCI feature is disabled.
-at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:89)
-at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
-at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
-at org.springframework.jdbc.core.JdbcTemplate.translateException(JdbcTemplate.java:1414)
-at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:388)
-at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:452)
-at com.google.edwmigration.dumper.application.dumper.task.JdbcSelectTask.doRun(JdbcSelectTask.java:102)
-at com.google.edwmigration.dumper.application.dumper.task.AbstractTask.run(AbstractTask.java:63)
-at com.google.edwmigration.dumper.application.dumper.task.Task.run(Task.java:89)
-at com.google.edwmigration.dumper.application.dumper.MetadataDumper.run(MetadataDumper.java:130)
-at com.google.edwmigration.dumper.application.dumper.MetadataDumper.run(MetadataDumper.java:76)
-at com.google.edwmigration.dumper.application.dumper.MetadataDumper.main(MetadataDumper.java:167)
-Caused by: java.sql.SQLException: [Teradata Database] [TeraJDBC 16.20.00.12] [Error 9719] [SQLState HY000] QVCI feature is disabled.
-at com.teradata.jdbc.jdbc_4.util.ErrorFactory.makeDatabaseSQLException(ErrorFactory.java:309)
-at com.teradata.jdbc.jdbc_4.statemachine.ReceiveInitSubState.action(ReceiveInitSubState.java:103)
-at com.teradata.jdbc.jdbc_4.statemachine.StatementReceiveState.subStateMachine(StatementReceiveState.java:311)
-at com.teradata.jdbc.jdbc_4.statemachine.StatementReceiveState.action(StatementReceiveState.java:200)
-at com.teradata.jdbc.jdbc_4.statemachine.StatementController.runBody(StatementController.java:137)
-at com.teradata.jdbc.jdbc_4.statemachine.StatementController.run(StatementController.java:128)
-at com.teradata.jdbc.jdbc_4.TDStatement.executeStatement(TDStatement.java:389)
-at com.teradata.jdbc.jdbc_4.TDStatement.executeStatement(TDStatement.java:331)
-at com.teradata.jdbc.jdbc_4.TDStatement.doNonPrepExecuteQuery(TDStatement.java:319)
-at com.teradata.jdbc.jdbc_4.TDStatement.executeQuery(TDStatement.java:1121)
-at org.springframework.jdbc.core.JdbcTemplate$1QueryStatementCallback.doInStatement(JdbcTemplate.java:439)
-at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:376)
-... 7 common frames omitted
- */
+}