Skip to content

Commit 33b070b

Browse files
ishmum123ishmum123
authored
format teradata connectors (#168)
1 parent cc01a12 commit 33b070b

File tree

5 files changed

+704
-607
lines changed

5 files changed

+704
-607
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java‎

Lines changed: 148 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -41,148 +41,175 @@
4141
import org.slf4j.Logger;
4242
import org.slf4j.LoggerFactory;
4343

44-
/**
45-
*
46-
*/
44+
/** */
4745
@AutoService({Connector.class, LogsConnector.class})
4846
@Description("Dumps logs from Teradata version <=14.")
4947
@RespectsArgumentQueryLogDays
5048
@RespectsArgumentQueryLogStart
5149
@RespectsArgumentQueryLogEnd
5250
public class Teradata14LogsConnector extends TeradataLogsConnector {
5351

54-
private static final Logger LOG = LoggerFactory.getLogger(Teradata14LogsConnector.class);
55-
56-
@VisibleForTesting
57-
static final List<String> EXPRESSIONS_LSQL_TBL = enumNames("ST.", TeradataLogsDumpFormat.HeaderLSql.class);
58-
59-
@VisibleForTesting
60-
static final List<String> EXPRESSIONS_LOG_TBL = enumNames("L.", TeradataLogsDumpFormat.HeaderLog.class);
61-
62-
private static List<String> enumNames(String prefix, Class< ? extends Enum<?>> en) {
63-
Enum<?> v[] = en.getEnumConstants();
64-
List<String> ret = new ArrayList<>(v.length);
65-
for (Enum<?> h : v)
66-
ret.add(prefix + h.name());
67-
return ret;
68-
}
69-
70-
public Teradata14LogsConnector() {
71-
super("teradata14-logs");
52+
private static final Logger LOG = LoggerFactory.getLogger(Teradata14LogsConnector.class);
53+
54+
@VisibleForTesting
55+
static final List<String> EXPRESSIONS_LSQL_TBL =
56+
enumNames("ST.", TeradataLogsDumpFormat.HeaderLSql.class);
57+
58+
@VisibleForTesting
59+
static final List<String> EXPRESSIONS_LOG_TBL =
60+
enumNames("L.", TeradataLogsDumpFormat.HeaderLog.class);
61+
62+
private static List<String> enumNames(String prefix, Class<? extends Enum<?>> en) {
63+
Enum<?> v[] = en.getEnumConstants();
64+
List<String> ret = new ArrayList<>(v.length);
65+
for (Enum<?> h : v) ret.add(prefix + h.name());
66+
return ret;
67+
}
68+
69+
public Teradata14LogsConnector() {
70+
super("teradata14-logs");
71+
}
72+
73+
private static class LSqlQueryFactory extends TeradataLogsJdbcTask {
74+
75+
public LSqlQueryFactory(
76+
String targetPath,
77+
SharedState state,
78+
String logTable,
79+
String queryTable,
80+
List<String> conditions,
81+
ZonedInterval interval) {
82+
super(targetPath, state, logTable, queryTable, conditions, interval);
7283
}
7384

74-
private static class LSqlQueryFactory extends TeradataLogsJdbcTask {
75-
76-
public LSqlQueryFactory(String targetPath, SharedState state, String logTable, String queryTable, List<String> conditions, ZonedInterval interval) {
77-
super(targetPath, state, logTable, queryTable, conditions, interval);
85+
@Override
86+
@Nonnull
87+
String getSql(@Nonnull Predicate<? super String> predicate) {
88+
StringBuilder buf = new StringBuilder("SELECT ");
89+
90+
String separator = "";
91+
for (String expression : EXPRESSIONS_LSQL_TBL) {
92+
buf.append(separator);
93+
if (predicate.test(expression)) {
94+
buf.append(expression);
95+
} else {
96+
buf.append("NULL");
7897
}
98+
separator = ", ";
99+
}
79100

80-
@Override
81-
@Nonnull
82-
String getSql(@Nonnull Predicate<? super String> predicate) {
83-
StringBuilder buf = new StringBuilder("SELECT ");
84-
85-
String separator = "";
86-
for (String expression : EXPRESSIONS_LSQL_TBL) {
87-
buf.append(separator);
88-
if (predicate.test(expression)) {
89-
buf.append(expression);
90-
} else {
91-
buf.append("NULL");
92-
}
93-
separator = ", ";
94-
}
95-
96-
buf.append(" FROM ").append(queryTable).append(" ST ");
97-
98-
buf.append(String.format("WHERE ST.CollectTimeStamp >= CAST('%s' AS TIMESTAMP)\n"
99-
+ "AND ST.CollectTimeStamp < CAST('%s' AS TIMESTAMP)\n",
100-
SQL_FORMAT.format(interval.getStart()), SQL_FORMAT.format(interval.getEndExclusive())));
101-
102-
for (String condition : conditions) {
103-
buf.append(" AND ").append(condition);
104-
}
105-
106-
return buf.toString().replace('\n', ' ');
107-
}
101+
buf.append(" FROM ").append(queryTable).append(" ST ");
108102

109-
}
103+
buf.append(
104+
String.format(
105+
"WHERE ST.CollectTimeStamp >= CAST('%s' AS TIMESTAMP)\n"
106+
+ "AND ST.CollectTimeStamp < CAST('%s' AS TIMESTAMP)\n",
107+
SQL_FORMAT.format(interval.getStart()),
108+
SQL_FORMAT.format(interval.getEndExclusive())));
110109

111-
private static class LogQueryFactory extends TeradataLogsJdbcTask {
112-
113-
public LogQueryFactory(String targetPath, SharedState state, String logTable, String queryTable, List<String> conditions, ZonedInterval interval) {
114-
super(targetPath, state, logTable, queryTable, conditions, interval);
115-
}
116-
117-
@Override
118-
@Nonnull
119-
String getSql(@Nonnull Predicate<? super String> predicate) {
120-
StringBuilder buf = new StringBuilder("SELECT ");
121-
122-
String separator = "";
123-
for (String expression : EXPRESSIONS_LOG_TBL) {
124-
buf.append(separator);
125-
if (predicate.test(expression)) {
126-
buf.append(expression);
127-
} else {
128-
buf.append("NULL");
129-
}
130-
separator = ", ";
131-
}
132-
133-
buf.append(" FROM ").append(logTable).append(" L ");
134-
135-
buf.append(String.format("WHERE L.StartTime >= CAST('%s' AS TIMESTAMP)\n"
136-
+ "AND L.StartTime < CAST('%s' AS TIMESTAMP)\n",
137-
SQL_FORMAT.format(interval.getStart()), SQL_FORMAT.format(interval.getEndExclusive())));
138-
139-
for (String condition : conditions) {
140-
buf.append(" AND ").append(condition);
141-
}
142-
143-
return buf.toString().replace('\n', ' ');
144-
}
110+
for (String condition : conditions) {
111+
buf.append(" AND ").append(condition);
112+
}
145113

114+
return buf.toString().replace('\n', ' ');
115+
}
116+
}
117+
118+
private static class LogQueryFactory extends TeradataLogsJdbcTask {
119+
120+
public LogQueryFactory(
121+
String targetPath,
122+
SharedState state,
123+
String logTable,
124+
String queryTable,
125+
List<String> conditions,
126+
ZonedInterval interval) {
127+
super(targetPath, state, logTable, queryTable, conditions, interval);
146128
}
147129

148130
@Override
149-
public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments arguments) throws MetadataDumperUsageException {
150-
out.add(new DumpMetadataTask(arguments, FORMAT_NAME));
151-
out.add(new FormatTask(FORMAT_NAME));
152-
153-
String logTable = DEF_LOG_TABLE;
154-
String queryTable = DEF_QUERY_TABLE;
155-
List<String> alternates = arguments.getQueryLogAlternates();
156-
if (!alternates.isEmpty()) {
157-
if (alternates.size() != 2)
158-
throw new MetadataDumperUsageException("Alternate query log tables must be given as a pair; you specified: " + alternates);
159-
logTable = alternates.get(0);
160-
queryTable = alternates.get(1);
131+
@Nonnull
132+
String getSql(@Nonnull Predicate<? super String> predicate) {
133+
StringBuilder buf = new StringBuilder("SELECT ");
134+
135+
String separator = "";
136+
for (String expression : EXPRESSIONS_LOG_TBL) {
137+
buf.append(separator);
138+
if (predicate.test(expression)) {
139+
buf.append(expression);
140+
} else {
141+
buf.append("NULL");
161142
}
143+
separator = ", ";
144+
}
162145

163-
// if the user specifies an earliest start time there will be extraneous empty dump files
164-
// because we always iterate over the full 7 trailing days; maybe it's worth
165-
// preventing that in the future. To do that, we should require getQueryLogEarliestTimestamp()
166-
// to parse and return an ISO instant, not a database-server-specific format.
167-
List<String> lSqlConditions = new ArrayList<>();
168-
List<String> logConditions = new ArrayList<>();
169-
if (!StringUtils.isBlank(arguments.getQueryLogEarliestTimestamp())) {
170-
lSqlConditions.add("ST.CollectTimeStamp >= " + arguments.getQueryLogEarliestTimestamp());
171-
logConditions.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp());
172-
}
146+
buf.append(" FROM ").append(logTable).append(" L ");
173147

174-
// Beware of Teradata SQLSTATE HY000. See issue #4126.
175-
// Most likely caused by some operation (equality?) being performed on a datum which is too long for a varchar.
176-
ZonedIntervalIterable intervals = ZonedIntervalIterable.forConnectorArguments(arguments);
177-
LOG.info("Exporting query log for " + intervals);
178-
SharedState state = new SharedState();
179-
for (ZonedInterval interval : intervals) {
180-
String LSqlfile = ZIP_ENTRY_PREFIX_LSQL + DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(interval.getStartUTC()) + ".csv";
181-
out.add(new LSqlQueryFactory(LSqlfile, state, logTable, queryTable, lSqlConditions, interval).withHeaderClass(TeradataLogsDumpFormat.HeaderLSql.class));
148+
buf.append(
149+
String.format(
150+
"WHERE L.StartTime >= CAST('%s' AS TIMESTAMP)\n"
151+
+ "AND L.StartTime < CAST('%s' AS TIMESTAMP)\n",
152+
SQL_FORMAT.format(interval.getStart()),
153+
SQL_FORMAT.format(interval.getEndExclusive())));
182154

183-
String LOGfile = ZIP_ENTRY_PREFIX_LOG + DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(interval.getStartUTC()) + ".csv";
184-
out.add(new LogQueryFactory(LOGfile, state, logTable, queryTable, logConditions, interval).withHeaderClass(TeradataLogsDumpFormat.HeaderLog.class));
155+
for (String condition : conditions) {
156+
buf.append(" AND ").append(condition);
157+
}
185158

186-
}
159+
return buf.toString().replace('\n', ' ');
160+
}
161+
}
162+
163+
@Override
164+
public void addTasksTo(List<? super Task<?>> out, @Nonnull ConnectorArguments arguments)
165+
throws MetadataDumperUsageException {
166+
out.add(new DumpMetadataTask(arguments, FORMAT_NAME));
167+
out.add(new FormatTask(FORMAT_NAME));
168+
169+
String logTable = DEF_LOG_TABLE;
170+
String queryTable = DEF_QUERY_TABLE;
171+
List<String> alternates = arguments.getQueryLogAlternates();
172+
if (!alternates.isEmpty()) {
173+
if (alternates.size() != 2)
174+
throw new MetadataDumperUsageException(
175+
"Alternate query log tables must be given as a pair; you specified: " + alternates);
176+
logTable = alternates.get(0);
177+
queryTable = alternates.get(1);
178+
}
179+
180+
// if the user specifies an earliest start time there will be extraneous empty dump files
181+
// because we always iterate over the full 7 trailing days; maybe it's worth
182+
// preventing that in the future. To do that, we should require getQueryLogEarliestTimestamp()
183+
// to parse and return an ISO instant, not a database-server-specific format.
184+
List<String> lSqlConditions = new ArrayList<>();
185+
List<String> logConditions = new ArrayList<>();
186+
if (!StringUtils.isBlank(arguments.getQueryLogEarliestTimestamp())) {
187+
lSqlConditions.add("ST.CollectTimeStamp >= " + arguments.getQueryLogEarliestTimestamp());
188+
logConditions.add("L.StartTime >= " + arguments.getQueryLogEarliestTimestamp());
189+
}
190+
191+
// Beware of Teradata SQLSTATE HY000. See issue #4126.
192+
// Most likely caused by some operation (equality?) being performed on a datum which is too long
193+
// for a varchar.
194+
ZonedIntervalIterable intervals = ZonedIntervalIterable.forConnectorArguments(arguments);
195+
LOG.info("Exporting query log for " + intervals);
196+
SharedState state = new SharedState();
197+
for (ZonedInterval interval : intervals) {
198+
String LSqlfile =
199+
ZIP_ENTRY_PREFIX_LSQL
200+
+ DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(interval.getStartUTC())
201+
+ ".csv";
202+
out.add(
203+
new LSqlQueryFactory(LSqlfile, state, logTable, queryTable, lSqlConditions, interval)
204+
.withHeaderClass(TeradataLogsDumpFormat.HeaderLSql.class));
205+
206+
String LOGfile =
207+
ZIP_ENTRY_PREFIX_LOG
208+
+ DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(interval.getStartUTC())
209+
+ ".csv";
210+
out.add(
211+
new LogQueryFactory(LOGfile, state, logTable, queryTable, logConditions, interval)
212+
.withHeaderClass(TeradataLogsDumpFormat.HeaderLog.class));
187213
}
214+
}
188215
}

0 commit comments

Comments
 (0)