Skip to content

Commit ca1c0e9

Browse files
dawidxcdawidxc
authored
Use query builder for DatabasesV query (#315)
Refactoring that replaces DatabasesV query generation from StringBuilder to query builder. The change also fixes the issue with the missing `AND` operator in the `WHERE` clause.
1 parent 10f7640 commit ca1c0e9

File tree

19 files changed

+708
-68
lines changed

19 files changed

+708
-68
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/MetadataQueryGenerator.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Copyright 2022-2023 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.edwmigration.dumper.application.dumper.connector.teradata;
18+
19+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.and;
20+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.eq;
21+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.identifier;
22+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.projection;
23+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.selectAll;
24+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.star;
25+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.stringLiteral;
26+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.subquery;
27+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression.select;
28+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression.selectTop;
29+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression.union;
30+
31+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression;
32+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.OrderBySpec;
33+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.OrderBySpec.Direction;
34+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression;
35+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression.FromClauseStepBuilder;
36+
import java.util.Optional;
37+
import java.util.OptionalLong;
38+
39+
/** Generator of queries used during dumping metadata from Teradata. */
40+
class MetadataQueryGenerator {
41+
42+
static final String DBC_INFO_QUERY =
43+
select(
44+
stringLiteral("teradata").as("dialect"),
45+
identifier("InfoData").as("version"),
46+
identifier("CURRENT_TIMESTAMP").as("export_time"))
47+
.from("dbc.dbcinfo")
48+
.where(eq(identifier("InfoKey"), stringLiteral("VERSION")))
49+
.serialize();
50+
51+
static String createSelectForDatabasesV(
52+
OptionalLong userRows, OptionalLong dbRows, Optional<Expression> condition) {
53+
String tableName = "DBC.DatabasesV";
54+
if (!userRows.isPresent() && !dbRows.isPresent()) {
55+
FromClauseStepBuilder partialQuery = select("%s").from(tableName);
56+
return condition.map(partialQuery::where).orElse(partialQuery).serialize();
57+
}
58+
SelectExpression usersSelect = createSingleDbKindSelectFromDatabasesV("U", userRows, condition);
59+
SelectExpression dbsSelect = createSingleDbKindSelectFromDatabasesV("D", dbRows, condition);
60+
return select("%s")
61+
.from(
62+
subquery(
63+
union(
64+
selectAll().from(subquery(usersSelect)).as("users").build(),
65+
selectAll().from(subquery(dbsSelect)).as("dbs").build())))
66+
.as("t")
67+
.serialize();
68+
}
69+
70+
private static SelectExpression createSingleDbKindSelectFromDatabasesV(
71+
String dbKind, OptionalLong rowCount, Optional<Expression> condition) {
72+
String tableName = "DBC.DatabasesV";
73+
Expression dbKindCondition = eq(identifier("DBKind"), stringLiteral(dbKind));
74+
Expression processedCondition =
75+
condition
76+
.<Expression>map(innerCondition -> and(innerCondition, dbKindCondition))
77+
.orElse(dbKindCondition);
78+
if (rowCount.isPresent()) {
79+
return selectTop(rowCount.getAsLong(), projection(star()))
80+
.from(tableName)
81+
.where(processedCondition)
82+
.orderBy(OrderBySpec.create(identifier("PermSpace"), Direction.DESC));
83+
} else {
84+
return selectAll().from(tableName).where(processedCondition).build();
85+
}
86+
}
87+
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataMetadataConnector.java‎

Lines changed: 13 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@
1616
*/
1717
package com.google.edwmigration.dumper.application.dumper.connector.teradata;
1818

19+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.MetadataQueryGenerator.DBC_INFO_QUERY;
1920
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.formatQuery;
2021
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.optionalIf;
21-
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.eq;
2222
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.identifier;
23-
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.stringLiteral;
24-
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression.select;
23+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.TeradataSelectBuilder.in;
2524
import static java.util.stream.Collectors.joining;
2625
import static java.util.stream.Collectors.toList;
2726

@@ -36,6 +35,7 @@
3635
import com.google.edwmigration.dumper.application.dumper.connector.Connector;
3736
import com.google.edwmigration.dumper.application.dumper.connector.ConnectorProperty;
3837
import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector;
38+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression;
3939
import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask;
4040
import com.google.edwmigration.dumper.application.dumper.task.FormatTask;
4141
import com.google.edwmigration.dumper.application.dumper.task.Task;
@@ -117,6 +117,10 @@ public TeradataMetadataConnector() {
117117
@Override
118118
public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
119119
throws MetadataDumperUsageException {
120+
Optional<Expression> databaseNameCondition =
121+
optionalIf(
122+
!arguments.getDatabases().isEmpty(),
123+
() -> in(identifier("DatabaseName"), arguments.getDatabases()));
120124
String whereDatabaseNameClause =
121125
new SqlBuilder()
122126
.withWhereInVals("\"DatabaseName\"", arguments.getDatabases())
@@ -141,22 +145,14 @@ public void addTasksTo(List<? super Task<?>> out, ConnectorArguments arguments)
141145

142146
out.add(
143147
new TeradataJdbcSelectTask(
144-
VersionFormat.ZIP_ENTRY_NAME,
145-
TaskCategory.OPTIONAL,
146-
select(
147-
stringLiteral("teradata").as("dialect"),
148-
identifier("InfoData").as("version"),
149-
identifier("CURRENT_TIMESTAMP").as("export_time"))
150-
.from("dbc.dbcinfo")
151-
.where(eq(identifier("InfoKey"), stringLiteral("VERSION")))
152-
.serialize()));
148+
VersionFormat.ZIP_ENTRY_NAME, TaskCategory.OPTIONAL, DBC_INFO_QUERY));
153149

154150
// This is theoretically more reliable than ColumnsV, but if we are bandwidth limited, we should
155151
// risk taking ColumnsV only.
156152
// out.add(new JdbcSelectTask(ColumnsFormat.ZIP_ENTRY_NAME, // Was: teradata.columns.csv
157153
// "SELECT \"DatabaseName\", \"TableName\", \"ColumnId\", \"ColumnName\", \"ColumnType\" FROM
158154
// DBC.Columns" + whereDatabaseNameClause + " ;"));
159-
out.add(createTaskForDatabasesV(whereDatabaseNameClause, arguments));
155+
out.add(createTaskForDatabasesV(arguments, databaseNameCondition));
160156
// out.add(new TeradataJdbcSelectTask("td.dbc.Tables.others.csv", "SELECT * FROM DBC.Tables
161157
// WHERE TableKind <> 'F' ORDER BY 1,2,3,4;"));
162158
// out.add(new TeradataJdbcSelectTask("td.dbc.Tables.functions.csv", "SELECT * FROM DBC.Tables
@@ -279,34 +275,16 @@ private static String escapeStringLiteral(String s) {
279275
}
280276

281277
private TeradataJdbcSelectTask createTaskForDatabasesV(
282-
String whereDatabaseNameClause, ConnectorArguments arguments)
278+
ConnectorArguments arguments, Optional<Expression> databaseNameCondition)
283279
throws MetadataDumperUsageException {
284-
StringBuilder query = new StringBuilder();
285280
OptionalLong userRows =
286281
parseMaxRows(arguments, TeradataMetadataConnectorProperties.DATABASES_V_USERS_MAX_ROWS);
287282
OptionalLong dbRows =
288283
parseMaxRows(arguments, TeradataMetadataConnectorProperties.DATABASES_V_DBS_MAX_ROWS);
289-
query.append("SELECT %s FROM ");
290-
if (!userRows.isPresent() && !dbRows.isPresent()) {
291-
query.append(" DBC.DatabasesV ").append(whereDatabaseNameClause);
292-
} else {
293-
query.append(" (SELECT * FROM ( ");
294-
appendSelect(
295-
query,
296-
userRows,
297-
" * FROM DBC.DatabasesV " + concatWhere(whereDatabaseNameClause, " DBKind='U' "),
298-
" ORDER BY PermSpace DESC ");
299-
query.append(" ) AS users UNION SELECT * FROM (");
300-
appendSelect(
301-
query,
302-
dbRows,
303-
" * FROM DBC.DatabasesV " + concatWhere(whereDatabaseNameClause, " DBKind='D' "),
304-
" ORDER BY PermSpace DESC ");
305-
query.append(" ) AS dbs) AS t");
306-
}
307-
query.append(';');
308284
return new TeradataJdbcSelectTask(
309-
DatabasesVFormat.ZIP_ENTRY_NAME, TaskCategory.REQUIRED, formatQuery(query.toString()));
285+
DatabasesVFormat.ZIP_ENTRY_NAME,
286+
TaskCategory.REQUIRED,
287+
MetadataQueryGenerator.createSelectForDatabasesV(userRows, dbRows, databaseNameCondition));
310288
}
311289

312290
private TeradataJdbcSelectTask createTaskForTableSizeV(
@@ -350,17 +328,6 @@ private static OptionalLong parseMaxRows(
350328
return PropertyParser.parseNumber(arguments, property, Range.atLeast(1L));
351329
}
352330

353-
private static String concatWhere(String whereClause, String condition) {
354-
StringBuilder result = new StringBuilder();
355-
if (whereClause.isEmpty()) {
356-
result.append(" WHERE ");
357-
} else {
358-
result.append(whereClause);
359-
}
360-
result.append(condition);
361-
return result.toString();
362-
}
363-
364331
private static void appendSelect(
365332
StringBuilder query, OptionalLong maxRowCountMaybe, String selectBody, String orderBy) {
366333
query.append(" SELECT ");

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import java.util.Optional;
2121
import java.util.function.Supplier;
2222

23-
class TeradataUtils {
23+
public class TeradataUtils {
2424

2525
public static <T> Optional<T> optionalIf(boolean condition, Supplier<T> supplier) {
2626
return condition ? Optional.of(supplier.get()) : Optional.empty();

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/query/ExpressionSerializer.java‎

Lines changed: 78 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,26 @@
1616
*/
1717
package com.google.edwmigration.dumper.application.dumper.connector.teradata.query;
1818

19+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.formatQuery;
20+
import static com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.OrderBySpec.Direction.DESC;
21+
1922
import com.google.common.collect.ImmutableList;
2023
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.BinaryExpression;
2124
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression;
2225
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Identifier;
26+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.InExpression;
27+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.NaryExpression;
28+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.OrderBySpec;
2329
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Projection;
2430
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectExpression;
31+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SelectSubqueryExpression;
2532
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SourceSpec;
2633
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.StringLiteral;
34+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SubqueryExpression;
35+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.SubquerySourceSpec;
2736
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.TableSourceSpec;
37+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.UnionExpression;
38+
import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.UnionSubqueryExpression;
2839
import java.util.Optional;
2940
import java.util.function.Consumer;
3041

@@ -44,10 +55,11 @@ public static String serialize(Expression expression) {
4455
String serialize() {
4556
serializedQuery = new StringBuilder();
4657
append(expression);
47-
return serializedQuery.toString();
58+
return formatQuery(serializedQuery.toString());
4859
}
4960

5061
private void append(Expression expr) {
62+
serializedQuery.append(' ');
5163
if (expr instanceof SelectExpression) {
5264
append((SelectExpression) expression);
5365
} else if (expr instanceof StringLiteral) {
@@ -56,22 +68,59 @@ private void append(Expression expr) {
5668
serializedQuery.append(((Identifier) expr).name());
5769
} else if (expr instanceof BinaryExpression) {
5870
append(((BinaryExpression) expr));
71+
} else if (expr instanceof UnionExpression) {
72+
append(((UnionExpression) expr));
73+
} else if (expr instanceof SubqueryExpression) {
74+
append(((SubqueryExpression) expr));
75+
} else if (expr instanceof NaryExpression) {
76+
append(((NaryExpression) expr));
77+
} else if (expr instanceof InExpression) {
78+
append(((InExpression) expr));
5979
} else {
6080
throw new IllegalArgumentException(String.format("Unsupported expression type: '%s'.", expr));
6181
}
6282
}
6383

84+
private void append(InExpression expr) {
85+
append(expr.lhs());
86+
serializedQuery.append(" IN (");
87+
appendCommaSeparated(expr.items(), this::append);
88+
serializedQuery.append(')');
89+
}
90+
91+
private void append(NaryExpression expr) {
92+
appendWithSeparators(expr.subexpressions(), expr.operator().name(), this::append);
93+
}
94+
6495
private void append(BinaryExpression expr) {
65-
appendSpaceIfNecessary();
6696
append(expr.lhs());
6797
serializedQuery.append(' ');
6898
serializedQuery.append(expr.operator());
6999
serializedQuery.append(' ');
70100
append(expr.rhs());
71101
}
72102

103+
private void append(SubqueryExpression expr) {
104+
serializedQuery.append('(');
105+
if (expr instanceof SelectSubqueryExpression) {
106+
append(((SelectSubqueryExpression) expr).selectExpression());
107+
} else if (expr instanceof UnionSubqueryExpression) {
108+
append(((UnionSubqueryExpression) expr).unionExpression());
109+
} else {
110+
throw new IllegalStateException(String.format("Unsupported subquery expression '%s'", expr));
111+
}
112+
serializedQuery.append(')');
113+
}
114+
115+
private void append(UnionExpression expr) {
116+
appendWithSeparators(expr.selectExpressions(), "UNION ALL", this::append);
117+
}
118+
73119
private void append(SelectExpression selectExpression) {
74120
serializedQuery.append("SELECT");
121+
selectExpression
122+
.topRowCount()
123+
.ifPresent(rowCount -> serializedQuery.append(" TOP ").append(rowCount));
75124
appendCommaSeparated(selectExpression.projections(), this::append);
76125
selectExpression
77126
.sourceSpec()
@@ -87,18 +136,36 @@ private void append(SelectExpression selectExpression) {
87136
serializedQuery.append(" WHERE");
88137
append(condition);
89138
});
139+
if (!selectExpression.orderBySpecs().isEmpty()) {
140+
serializedQuery.append(" ORDER BY");
141+
appendCommaSeparated(selectExpression.orderBySpecs(), this::append);
142+
}
143+
}
144+
145+
private void append(OrderBySpec orderBySpec) {
146+
append(orderBySpec.expression());
147+
if (orderBySpec.direction() == DESC) {
148+
serializedQuery.append(" DESC");
149+
}
90150
}
91151

92152
private void append(SourceSpec sourceSpec) {
153+
serializedQuery.append(' ');
93154
if (sourceSpec instanceof TableSourceSpec) {
94155
append((TableSourceSpec) sourceSpec);
156+
} else if (sourceSpec instanceof SubquerySourceSpec) {
157+
append((SubquerySourceSpec) sourceSpec);
95158
} else {
96159
throw new IllegalStateException(String.format("Unsupported source spec='%s'.", sourceSpec));
97160
}
98161
}
99162

163+
private void append(SubquerySourceSpec sourceSpec) {
164+
append(sourceSpec.subqueryExpression());
165+
append(sourceSpec.alias());
166+
}
167+
100168
private void append(TableSourceSpec tableSourceSpec) {
101-
serializedQuery.append(' ');
102169
append(tableSourceSpec.tableName());
103170
append(tableSourceSpec.alias());
104171
}
@@ -112,19 +179,21 @@ private void append(Optional<String> aliasMaybe) {
112179
aliasMaybe.ifPresent(alias -> serializedQuery.append(" AS ").append(alias));
113180
}
114181

115-
private void appendSpaceIfNecessary() {
116-
if (serializedQuery.length() > 0) {
117-
serializedQuery.append(' ');
118-
}
182+
private <T> void appendCommaSeparated(ImmutableList<T> list, Consumer<T> appender) {
183+
appendWithSeparators(list, ",", appender);
119184
}
120185

121-
private <T> void appendCommaSeparated(ImmutableList<T> list, Consumer<T> appender) {
186+
private <T> void appendWithSeparators(
187+
ImmutableList<T> list, String separator, Consumer<T> appender) {
122188
boolean first = true;
123189
for (T element : list) {
124190
if (first) {
125191
first = false;
126192
} else {
127-
serializedQuery.append(',');
193+
if (!separator.equals(",")) {
194+
serializedQuery.append(' ');
195+
}
196+
serializedQuery.append(separator);
128197
}
129198
serializedQuery.append(' ');
130199
appender.accept(element);

0 commit comments

Comments
 (0)