Skip to content

Commit 4ca18b4

Browse files
dawidxcdawidxc
authored
Add --max-databasesv-user-rows and --max-databasesv-db-rows parameters. (#154)
* Add --max-databasesv-user-rows and --max-databasesv-db-rows parameters. Add possibility to limit the number of rows extracted from DBC.DatabasesV, separately for user rows (DBKind='U') and database rows (DBKind='D'). * Add argument validation.
1 parent 1827794 commit 4ca18b4

File tree

3 files changed

+117
-12
lines changed

3 files changed

+117
-12
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/ConnectorArguments.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ public class ConnectorArguments extends DefaultArguments {
9999
public static final String OPT_SCHEMA = "schema";
100100
public static final String OPT_ASSESSMENT = "assessment";
101101
public static final String OPT_TERADATA_MAX_TABLESIZEV_ROWS = "max-tablesizev-rows";
102+
public static final String OPT_TERADATA_MAX_DATABASESV_USER_ROWS = "max-databasesv-user-rows";
103+
public static final String OPT_TERADATA_MAX_DATABASESV_DB_ROWS = "max-databasesv-db-rows";
102104
public static final String OPT_ORACLE_SID = "oracle-sid";
103105
public static final String OPT_ORACLE_SERVICE = "oracle-service";
104106

@@ -139,6 +141,16 @@ public class ConnectorArguments extends DefaultArguments {
139141
private final OptionSpec<Long> optionTeradataMaxTableSizeVRows =
140142
parser.accepts(OPT_TERADATA_MAX_TABLESIZEV_ROWS, TERADATA_MAX_TABLE_SIZE_V_ROWS_DESCRIPTION)
141143
.withRequiredArg().ofType(Long.class).describedAs("100000");
144+
public static final String TERADATA_MAX_DATABASES_V_USER_ROWS_DESCRIPTION =
145+
"Max number of user rows (DBKind='U') to extract from DBC.DatabasesV table (available for 'teradata' connector only)";
146+
private final OptionSpec<Long> optionTeradataMaxDatabasesVUserRows =
147+
parser.accepts(OPT_TERADATA_MAX_DATABASESV_USER_ROWS, TERADATA_MAX_DATABASES_V_USER_ROWS_DESCRIPTION)
148+
.withRequiredArg().ofType(Long.class).describedAs("100000");
149+
public static final String TERADATA_MAX_DATABASES_V_DB_ROWS_DESCRIPTION =
150+
"Max number of database rows (DBKind='D') to extract from DBC.DatabasesV table (available for 'teradata' connector only)";
151+
private final OptionSpec<Long> optionTeradataMaxDatabasesVDbRows =
152+
parser.accepts(OPT_TERADATA_MAX_DATABASESV_DB_ROWS, TERADATA_MAX_DATABASES_V_DB_ROWS_DESCRIPTION)
153+
.withRequiredArg().ofType(Long.class).describedAs("100000");
142154

143155
private final OptionSpec<String> optionUser = parser.accepts(OPT_USER, "Database username").withRequiredArg().describedAs("admin");
144156
private final OptionSpec<String> optionPass = parser.accepts(OPT_PASSWORD, "Database password, prompted if not provided").withOptionalArg().describedAs("sekr1t");
@@ -427,10 +439,22 @@ public boolean isAssessment() {
427439
}
428440

429441
public Optional<Long> getTeradataMaxTableSizeVRows() {
430-
if (!getOptions().has(optionTeradataMaxTableSizeVRows)) {
442+
return optionAsOptional(optionTeradataMaxTableSizeVRows);
443+
}
444+
445+
public Optional<Long> getTeradataMaxDatabasesVUserRows() {
446+
return optionAsOptional(optionTeradataMaxDatabasesVUserRows);
447+
}
448+
449+
public Optional<Long> getTeradataMaxDatabasesVDbRows() {
450+
return optionAsOptional(optionTeradataMaxDatabasesVDbRows);
451+
}
452+
453+
private <T> Optional<T> optionAsOptional(OptionSpec<T> spec) {
454+
if (!getOptions().has(spec)) {
431455
return Optional.empty();
432456
}
433-
return Optional.of(getOptions().valueOf(optionTeradataMaxTableSizeVRows));
457+
return Optional.of(getOptions().valueOf(spec));
434458
}
435459

436460
@Nonnull

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
*/
1717
package com.google.edwmigration.dumper.application.dumper;
1818

19+
import static com.google.edwmigration.dumper.application.dumper.ConnectorArguments.OPT_TERADATA_MAX_DATABASESV_DB_ROWS;
20+
import static com.google.edwmigration.dumper.application.dumper.ConnectorArguments.OPT_TERADATA_MAX_DATABASESV_USER_ROWS;
21+
import static com.google.edwmigration.dumper.application.dumper.ConnectorArguments.OPT_TERADATA_MAX_TABLESIZEV_ROWS;
22+
1923
import com.google.common.base.Preconditions;
2024
import com.google.common.base.Stopwatch;
2125
import com.google.common.collect.ImmutableMap;
@@ -50,6 +54,7 @@
5054
import java.util.Arrays;
5155
import java.util.List;
5256
import java.util.Map;
57+
import java.util.Optional;
5358
import java.util.ServiceLoader;
5459
import java.util.function.Function;
5560
import java.util.stream.Collectors;
@@ -94,6 +99,29 @@ private static String repeat(@Nonnull char c, @Nonnegative int n) {
9499
return new String(out);
95100
}
96101

102+
private static class ArgumentValidationException extends RuntimeException {
103+
ArgumentValidationException(String message) {
104+
super(message);
105+
}
106+
}
107+
108+
private static void validateArgumentGreaterThan(Optional<Long> valueMaybe, long minValue,
109+
String argumentName) {
110+
valueMaybe.ifPresent(value -> {
111+
if(value <= minValue) {
112+
throw new ArgumentValidationException(
113+
String.format("Argument for '%s' option must be greater than 0. Actual: '%s'.",
114+
argumentName, value));
115+
}
116+
});
117+
}
118+
119+
private static void validateArguments(ConnectorArguments arguments) throws ArgumentValidationException {
120+
validateArgumentGreaterThan(arguments.getTeradataMaxTableSizeVRows(), 0, OPT_TERADATA_MAX_TABLESIZEV_ROWS);
121+
validateArgumentGreaterThan(arguments.getTeradataMaxDatabasesVUserRows(), 0, OPT_TERADATA_MAX_DATABASESV_USER_ROWS);
122+
validateArgumentGreaterThan(arguments.getTeradataMaxDatabasesVDbRows(), 0, OPT_TERADATA_MAX_DATABASESV_DB_ROWS);
123+
}
124+
97125
public void run(@Nonnull String... args) throws Exception {
98126
ConnectorArguments arguments = new ConnectorArguments(args);
99127

@@ -109,6 +137,13 @@ public void run(@Nonnull String... args) throws Exception {
109137
return;
110138
}
111139

140+
try {
141+
validateArguments(arguments);
142+
} catch (ArgumentValidationException ex) {
143+
LOG.error("ERROR: Validation failed. {}", ex.getMessage());
144+
return;
145+
}
146+
112147
try {
113148
run(connector, arguments);
114149
} finally {

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataMetadataConnector.java‎

Lines changed: 56 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description;
3232
import com.google.edwmigration.dumper.application.dumper.utils.SqlBuilder;
3333
import com.google.edwmigration.dumper.plugin.lib.dumper.spi.TeradataMetadataDumpFormat;
34+
import java.util.Optional;
3435
import org.slf4j.Logger;
3536
import org.slf4j.LoggerFactory;
3637

@@ -44,6 +45,10 @@
4445
@RespectsArgumentAssessment
4546
@RespectsInput(order = 450, arg = ConnectorArguments.OPT_TERADATA_MAX_TABLESIZEV_ROWS,
4647
description = ConnectorArguments.TERADATA_MAX_TABLE_SIZE_V_ROWS_DESCRIPTION)
48+
@RespectsInput(order = 450, arg = ConnectorArguments.OPT_TERADATA_MAX_DATABASESV_USER_ROWS,
49+
description = ConnectorArguments.TERADATA_MAX_DATABASES_V_USER_ROWS_DESCRIPTION)
50+
@RespectsInput(order = 450, arg = ConnectorArguments.OPT_TERADATA_MAX_DATABASESV_DB_ROWS,
51+
description = ConnectorArguments.TERADATA_MAX_DATABASES_V_DB_ROWS_DESCRIPTION)
4752
public class TeradataMetadataConnector extends AbstractTeradataConnector implements MetadataConnector, TeradataMetadataDumpFormat {
4853

4954
@SuppressWarnings("UnusedVariable")
@@ -78,9 +83,7 @@ public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
7883
// This is theoretically more reliable than ColumnsV, but if we are bandwidth limited, we should risk taking ColumnsV only.
7984
// out.add(new JdbcSelectTask(ColumnsFormat.ZIP_ENTRY_NAME, // Was: teradata.columns.csv
8085
// "SELECT \"DatabaseName\", \"TableName\", \"ColumnId\", \"ColumnName\", \"ColumnType\" FROM DBC.Columns" + whereDatabaseNameClause + " ;"));
81-
out.add(new TeradataJdbcSelectTask(DatabasesVFormat.ZIP_ENTRY_NAME,
82-
TaskCategory.REQUIRED,
83-
"SELECT %s FROM DBC.DatabasesV" + whereDatabaseNameClause + " ;"));
86+
out.add(createTaskForDatabasesV(whereDatabaseNameClause, arguments));
8487
//out.add(new TeradataJdbcSelectTask("td.dbc.Tables.others.csv", "SELECT * FROM DBC.Tables WHERE TableKind <> 'F' ORDER BY 1,2,3,4;"));
8588
//out.add(new TeradataJdbcSelectTask("td.dbc.Tables.functions.csv", "SELECT * FROM DBC.Tables WHERE TableKind = 'F' ORDER BY 1,2,3,4;"));
8689
// TODO: This contains RequestText for views, which doesn't tell us the "current" database at the point the view was defined.
@@ -140,25 +143,68 @@ public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
140143
}
141144
}
142145

146+
private TeradataJdbcSelectTask createTaskForDatabasesV(
147+
String whereDatabaseNameClause,
148+
ConnectorArguments arguments) {
149+
StringBuilder query = new StringBuilder();
150+
Optional<Long> userRows = arguments.getTeradataMaxDatabasesVUserRows();
151+
Optional<Long> dbRows = arguments.getTeradataMaxDatabasesVDbRows();
152+
query.append("SELECT %s FROM ");
153+
if(!userRows.isPresent() && !dbRows.isPresent()) {
154+
query.append(" DBC.DatabasesV ").append(whereDatabaseNameClause);
155+
} else {
156+
query.append(" (SELECT * FROM ( ");
157+
appendSelect(query, userRows,
158+
" * FROM DBC.DatabasesV "
159+
+ concatWhere(whereDatabaseNameClause, " DBKind='U' "), " ORDER BY PermSpace DESC ");
160+
query.append(" ) AS users UNION SELECT * FROM (");
161+
appendSelect(query, dbRows,
162+
" * FROM DBC.DatabasesV "
163+
+ concatWhere(whereDatabaseNameClause, " DBKind='D' "), " ORDER BY PermSpace DESC ");
164+
query.append(" ) AS dbs) AS t");
165+
}
166+
query.append(';');
167+
return new TeradataJdbcSelectTask(DatabasesVFormat.ZIP_ENTRY_NAME,
168+
TaskCategory.REQUIRED, formatQuery(query.toString()));
169+
}
170+
143171
private TeradataJdbcSelectTask createTaskForTableSizeV(
144172
String whereDataBaseNameClause,
145173
ConnectorArguments arguments) {
146174
StringBuilder query = new StringBuilder();
147-
query.append("SELECT ");
148-
arguments.getTeradataMaxTableSizeVRows().ifPresent(maxRows -> query.append("TOP ").append(maxRows));
149175
// TableSizeV contains a row per each VProc/AMP, so it can grow significantly for large dbs.
150176
// Hence, we aggregate before dumping.
151177
// See recommended usage
152178
// https://docs.teradata.com/r/Teradata-VantageTM-Data-Dictionary/March-2019/Views-Reference/TableSizeV-X/Examples-Using-TableSizeV
153-
query.append(" DataBaseName, AccountName, TableName, SUM(CurrentPerm) CurrentPerm, SUM(PeakPerm) PeakPerm FROM DBC.TableSizeV ")
154-
.append(whereDataBaseNameClause)
155-
.append(" GROUP BY 1,2,3");
156-
arguments.getTeradataMaxTableSizeVRows().ifPresent(unused -> query.append(" ORDER BY 4 DESC"));
179+
appendSelect(query, arguments.getTeradataMaxTableSizeVRows(),
180+
" DataBaseName, AccountName, TableName, SUM(CurrentPerm) CurrentPerm, SUM(PeakPerm) PeakPerm FROM DBC.TableSizeV "
181+
+ whereDataBaseNameClause + " GROUP BY 1,2,3 ", " ORDER BY 4 DESC ");
157182
query.append(';');
158183
return new TeradataJdbcSelectTask(
159184
TableSizeVFormat.ZIP_ENTRY_NAME,
160185
TaskCategory.OPTIONAL,
161-
query.toString());
186+
formatQuery(query.toString()));
162187
}
163188

189+
private static String concatWhere(String whereClause, String condition) {
190+
StringBuilder result = new StringBuilder();
191+
if(whereClause.isEmpty()) {
192+
result.append(" WHERE ");
193+
} else {
194+
result.append(whereClause);
195+
}
196+
result.append(condition);
197+
return result.toString();
198+
}
199+
200+
private static void appendSelect(StringBuilder query, Optional<Long> maxRowCountMaybe, String selectBody, String orderBy) {
201+
query.append(" SELECT ");
202+
maxRowCountMaybe.ifPresent(maxRowCount -> query.append(" TOP ").append(maxRowCount).append(' '));
203+
query.append(selectBody);
204+
maxRowCountMaybe.ifPresent(unused -> query.append(orderBy));
205+
}
206+
207+
private static String formatQuery(String query) {
208+
return query.replaceAll("\\s+", " ").replaceAll("\\( ", "(").replaceAll(" \\)", ")").trim();
209+
}
164210
}

0 commit comments

Comments
 (0)