Skip to content

Commit 9162ae5

Browse files
dawidxcdawidxc
authored
Fix WHERE clause for non-assessment Teradata tasks (#341)
The WHERE clause generated for Teradata logs tasks (in `teradata-logs` connector without specifying the `--assessment` flag) contained repeated conditions `L.UserName <> 'DBC'`: ``` L.UserName <> 'DBC' AND L.UserName <> 'DBC' AND L.UserName <> 'DBC' AND ... ``` The `conditions` list was mistakenly modified inside the loop, which caused additional redundant conditions to be added to the query, if there was more than one time interval extracted. If there were too many such conditions, the query failed.
1 parent a5cdefc commit 9162ae5

File tree

7 files changed

+278
-26
lines changed

7 files changed

+278
-26
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
2323
import java.util.List;
2424
import java.util.OptionalLong;
25+
import java.util.Set;
2526
import java.util.function.Predicate;
2627
import javax.annotation.CheckForNull;
2728
import javax.annotation.Nonnull;
@@ -160,7 +161,7 @@ public TeradataAssessmentLogsJdbcTask(
160161
SharedState state,
161162
String logTable,
162163
String queryTable,
163-
List<String> conditions,
164+
Set<String> conditions,
164165
ZonedInterval interval,
165166
@CheckForNull String logDateColumn,
166167
OptionalLong maxSqlLength,

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.auto.service.AutoService;
2222
import com.google.common.collect.ImmutableList;
2323
import com.google.common.collect.ImmutableMap;
24+
import com.google.common.collect.ImmutableSet;
2425
import com.google.common.collect.Range;
2526
import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
2627
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
@@ -44,7 +45,6 @@
4445
import com.google.edwmigration.dumper.plugin.lib.dumper.spi.TeradataLogsDumpFormat;
4546
import java.time.Duration;
4647
import java.time.format.DateTimeFormatter;
47-
import java.util.ArrayList;
4848
import java.util.Arrays;
4949
import java.util.List;
5050
import java.util.OptionalLong;
@@ -189,13 +189,13 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
189189
logTable = alternates.get(0);
190190
queryTable = alternates.get(1);
191191
}
192-
List<String> conditions = new ArrayList<>();
192+
ImmutableSet.Builder<String> conditionsBuilder = ImmutableSet.builder();
193193
// if the user specifies an earliest start time there will be extraneous empty dump files
194194
// because we always iterate over the full 7 trailing days; maybe it's worth
195195
// preventing that in the future. To do that, we should require getQueryLogEarliestTimestamp()
196196
// to parse and return an ISO instant, not a database-server-specific format.
197197
if (!StringUtils.isBlank(arguments.getQueryLogEarliestTimestamp())) {
198-
conditions.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp());
198+
conditionsBuilder.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp());
199199
}
200200

201201
Duration rotationDuration = arguments.getQueryLogRotationFrequency();
@@ -211,6 +211,10 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
211211
OptionalLong maxSqlLength =
212212
PropertyParser.parseNumber(
213213
arguments, TeradataLogsConnectorProperty.MAX_SQL_LENGTH, MAX_SQL_LENGTH_RANGE);
214+
if (!isAssessment) {
215+
conditionsBuilder.add("L.UserName <> 'DBC'");
216+
}
217+
ImmutableSet<String> conditions = conditionsBuilder.build();
214218
for (ZonedInterval interval : intervals) {
215219
String file = createFilename(ZIP_ENTRY_PREFIX, interval);
216220
if (isAssessment) {
@@ -235,7 +239,6 @@ public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments ar
235239
utilityLogsTable,
236240
interval));
237241
} else {
238-
conditions.add("L.UserName <> 'DBC'");
239242
out.add(
240243
new TeradataLogsJdbcTask(
241244
file, queryLogsState, logTable, queryTable, conditions, interval)

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.google.common.base.Preconditions;
3030
import com.google.common.base.Predicates;
3131
import com.google.common.collect.ImmutableList;
32+
import com.google.common.collect.ImmutableSet;
3233
import com.google.common.io.ByteSink;
3334
import com.google.common.primitives.Ints;
3435
import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
@@ -43,9 +44,9 @@
4344
import java.sql.SQLException;
4445
import java.time.ZoneOffset;
4546
import java.time.format.DateTimeFormatter;
46-
import java.util.Collections;
4747
import java.util.List;
4848
import java.util.OptionalLong;
49+
import java.util.Set;
4950
import java.util.function.Function;
5051
import java.util.function.Predicate;
5152
import javax.annotation.CheckForNull;
@@ -100,18 +101,18 @@ public class TeradataLogsJdbcTask extends AbstractJdbcTask<Summary> {
100101
protected final SharedState state;
101102
protected final String logTable;
102103
protected final String queryTable;
103-
protected final List<String> conditions;
104+
protected final ImmutableSet<String> conditions;
104105
protected final ZonedInterval interval;
105106
@CheckForNull private final String logDateColumn;
106107
private final OptionalLong maxSqlLength;
107-
protected final List<String> orderBy;
108+
protected final ImmutableList<String> orderBy;
108109

109110
public TeradataLogsJdbcTask(
110111
@Nonnull String targetPath,
111112
SharedState state,
112113
String logTable,
113114
String queryTable,
114-
List<String> conditions,
115+
Set<String> conditions,
115116
ZonedInterval interval) {
116117
this(
117118
targetPath,
@@ -122,15 +123,15 @@ public TeradataLogsJdbcTask(
122123
interval,
123124
/* logDateColumn= */ null,
124125
/* maxSqlLength= */ OptionalLong.empty(),
125-
Collections.emptyList());
126+
/* orderBy= */ ImmutableList.of());
126127
}
127128

128129
protected TeradataLogsJdbcTask(
129130
@Nonnull String targetPath,
130131
SharedState state,
131132
String logTable,
132133
String queryTable,
133-
List<String> conditions,
134+
Set<String> conditions,
134135
ZonedInterval interval,
135136
@CheckForNull String logDateColumn,
136137
OptionalLong maxSqlLength,
@@ -139,11 +140,11 @@ protected TeradataLogsJdbcTask(
139140
this.state = Preconditions.checkNotNull(state, "SharedState was null.");
140141
this.logTable = logTable;
141142
this.queryTable = queryTable;
142-
this.conditions = conditions;
143+
this.conditions = ImmutableSet.copyOf(conditions);
143144
this.interval = interval;
144145
this.logDateColumn = logDateColumn;
145146
this.maxSqlLength = maxSqlLength;
146-
this.orderBy = orderBy;
147+
this.orderBy = ImmutableList.copyOf(orderBy);
147148
}
148149

149150
private static boolean isQueryTable(@Nonnull String expression) {
@@ -248,7 +249,7 @@ private String getSql(@Nonnull JdbcHandle handle) {
248249
buf.append(" ORDER BY ");
249250
Joiner.on(", ").appendTo(buf, orderBy);
250251
}
251-
return buf.toString().replace('\n', ' ');
252+
return formatQuery(buf.toString());
252253
}
253254

254255
private String createLogDateColumnConditionStr() {

‎dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractConnectorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public abstract class AbstractConnectorTest {
4040
@SuppressWarnings("UnusedVariable")
4141
private static final Logger LOG = LoggerFactory.getLogger(AbstractConnectorTest.class);
4242

43-
protected static enum SpecialTaskType {
43+
protected enum SpecialTaskType {
4444
Version,
4545
Arguments,
4646
Format

‎dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTaskTest.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static java.util.Collections.emptyList;
2121

2222
import com.google.common.collect.ImmutableList;
23+
import com.google.common.collect.ImmutableSet;
2324
import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
2425
import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
2526
import java.time.ZoneId;
@@ -52,7 +53,7 @@ public void getSql_success() {
5253
queryLogsState,
5354
"SampleQueryTable",
5455
"SampleSqlTable",
55-
/* conditions= */ emptyList(),
56+
/* conditions= */ ImmutableSet.of(),
5657
interval,
5758
/* logDateColumn= */ null,
5859
/* maxSqlLength= */ OptionalLong.empty(),
@@ -79,7 +80,7 @@ public void getSql_maxSqlLength() {
7980
queryLogsState,
8081
"SampleQueryTable",
8182
"SampleSqlTable",
82-
/* conditions= */ emptyList(),
83+
/* conditions= */ ImmutableSet.of(),
8384
interval,
8485
/* logDateColumn= */ null,
8586
/* maxSqlLength= */ OptionalLong.of(20000),
@@ -114,7 +115,7 @@ public void getSql_noSecondTable() {
114115
queryLogsState,
115116
"SampleQueryTable",
116117
"SampleSqlTable",
117-
/* conditions= */ emptyList(),
118+
/* conditions= */ ImmutableSet.of(),
118119
interval,
119120
/* logDateColumn= */ null,
120121
/* maxSqlLength= */ OptionalLong.empty(),
@@ -141,7 +142,7 @@ public void getSql_noSecondTableWithLogDateColumn() {
141142
queryLogsState,
142143
"SampleQueryTable",
143144
"SampleSqlTable",
144-
/* conditions= */ emptyList(),
145+
/* conditions= */ ImmutableSet.of(),
145146
interval,
146147
"SampleLogDate", /* orderBy */
147148
/* maxSqlLength= */ OptionalLong.empty(),
@@ -169,7 +170,7 @@ public void getSql_noSecondTableWithCondition() {
169170
queryLogsState,
170171
"SampleQueryTable",
171172
"SampleSqlTable",
172-
/* conditions= */ ImmutableList.of("L.QueryID=7"),
173+
/* conditions= */ ImmutableSet.of("L.QueryID=7"),
173174
interval,
174175
/* logDateColumn= */ null,
175176
/* maxSqlLength= */ OptionalLong.empty(),
@@ -196,7 +197,7 @@ public void getSql_noSecondTableWithOrderBy() {
196197
queryLogsState,
197198
"SampleQueryTable",
198199
"SampleSqlTable",
199-
/* conditions= */ emptyList(),
200+
/* conditions= */ ImmutableSet.of(),
200201
interval,
201202
/* logDateColumn= */ null,
202203
/* maxSqlLength= */ OptionalLong.empty(),
@@ -224,7 +225,7 @@ public void getSql_withLogDateColumn() {
224225
queryLogsState,
225226
"SampleQueryTable",
226227
"SampleSqlTable",
227-
/* conditions= */ emptyList(),
228+
/* conditions= */ ImmutableSet.of(),
228229
interval,
229230
"SampleLogDate", /* orderBy */
230231
/* maxSqlLength= */ OptionalLong.empty(),
@@ -253,7 +254,7 @@ public void getSql_withLogDateColumnAndMaxSqlLength() {
253254
queryLogsState,
254255
"SampleQueryTable",
255256
"SampleSqlTable",
256-
/* conditions= */ emptyList(),
257+
/* conditions= */ ImmutableSet.of(),
257258
interval,
258259
"SampleLogDate", /* orderBy */
259260
/* maxSqlLength= */ OptionalLong.of(20000),
@@ -292,7 +293,7 @@ public void getSql_fullQuery() {
292293
queryLogsState,
293294
"SampleQueryTable",
294295
"SampleSqlTable",
295-
ImmutableList.of("QueryID=7", "QueryText LIKE '%abc%'"),
296+
ImmutableSet.of("QueryID=7", "QueryText LIKE '%abc%'"),
296297
interval,
297298
"SampleLogDate",
298299
/* maxSqlLength= */ OptionalLong.empty(),

0 commit comments

Comments
 (0)