Skip to content

Commit 38879e8

Browse files
paolomorandinipaolomorandini
committed
[Snowflake Connector] Add tests for connector's SQL overrides.
1 parent 434ab74 commit 38879e8

File tree

8 files changed

+150
-5
lines changed

8 files changed

+150
-5
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public class SnowflakeMetadataConnector extends AbstractSnowflakeConnector
5050
@SuppressWarnings("UnusedVariable")
5151
private static final Logger logger = LoggerFactory.getLogger(SnowflakeMetadataConnector.class);
5252

53-
private enum PropertyAction {
53+
enum PropertyAction {
5454
QUERY("query", "query"),
5555
WHERE("where", "where condition to append to query");
5656

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeTaskUtil.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,21 +24,25 @@
2424
@ParametersAreNonnullByDefault
2525
final class SnowflakeTaskUtil {
2626

27+
private static final String EMPTY_WHERE_CLAUSE = "";
28+
2729
static AbstractJdbcTask<Summary> withFilter(
2830
String format,
2931
String schemaName,
3032
String zipEntryName,
3133
String whereClause,
3234
Class<? extends Enum<?>> header) {
35+
if (format.contains(" WHERE ")) {
36+
whereClause = EMPTY_WHERE_CLAUSE;
37+
}
3338
String sql = String.format(format, schemaName, whereClause);
3439
return new JdbcSelectTask(zipEntryName, sql).withHeaderClass(header);
3540
}
3641

3742
static AbstractJdbcTask<Summary> withNoFilter(
3843
String format, String schemaName, String zipEntryName, Class<? extends Enum<?>> header) {
3944
// required, because the format string parameter takes two args
40-
String whereClause = "";
41-
String sql = String.format(format, schemaName, whereClause);
45+
String sql = String.format(format, schemaName, EMPTY_WHERE_CLAUSE);
4246
return new JdbcSelectTask(zipEntryName, sql).withHeaderClass(header);
4347
}
4448

‎dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/snowflake/SnowflakeMetadataConnectorTest.java‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
2222
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
2323
import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector;
24+
import com.google.edwmigration.dumper.application.dumper.connector.snowflake.SnowflakeMetadataConnector.PropertyAction;
2425
import com.google.edwmigration.dumper.application.dumper.task.JdbcSelectTask;
2526
import com.google.edwmigration.dumper.application.dumper.task.Task;
2627
import com.google.edwmigration.dumper.plugin.lib.dumper.spi.CoreMetadataDumpFormat;
@@ -34,6 +35,7 @@
3435
import java.util.List;
3536
import java.util.Map;
3637
import javax.annotation.Nonnull;
38+
import org.apache.commons.lang3.ArrayUtils;
3739
import org.junit.Assert;
3840
import org.junit.Assume;
3941
import org.junit.Test;
@@ -154,10 +156,93 @@ public void connector_generatesExpectedSql() throws IOException {
154156
}
155157
}
156158

157-
private static Map<String, String> collectSqlStatements() throws IOException {
159+
@Test
160+
public void connector_assessment_generatesExpectedSql() throws IOException {
161+
Map<String, String> actualSqls = collectSqlStatements("--assessment");
162+
TaskSqlMap expectedSqls =
163+
CoreMetadataDumpFormat.MAPPER.readValue(
164+
Resources.toString(
165+
Resources.getResource("connector/snowflake/assessment-jdbc-tasks-sql.yaml"),
166+
StandardCharsets.UTF_8),
167+
TaskSqlMap.class);
168+
169+
Assert.assertEquals(expectedSqls.size(), actualSqls.size());
170+
Assert.assertEquals(expectedSqls.keySet(), actualSqls.keySet());
171+
for (String name : expectedSqls.keySet()) {
172+
Assert.assertEquals(expectedSqls.get(name), actualSqls.get(name));
173+
}
174+
}
175+
176+
@Test
177+
public void connector_generatesExpectedSql_withQueryOverrides() throws IOException {
178+
for (PropertyAction propertyAction : PropertyAction.values()) {
179+
TaskSqlMap expectedSqls =
180+
CoreMetadataDumpFormat.MAPPER.readValue(
181+
Resources.toString(
182+
Resources.getResource(
183+
String.format(
184+
"connector/snowflake/jdbc-tasks-sql-override-%s.yaml",
185+
propertyAction.value)),
186+
StandardCharsets.UTF_8),
187+
TaskSqlMap.class);
188+
189+
List<String> overrideArguments = new ArrayList<>();
190+
for (MetadataView metadataView : MetadataView.values()) {
191+
overrideArguments.add(
192+
String.format(
193+
"-Dsnowflake.metadata.%1$s.%2$s=SQL_OVERRIDE(%1$s, %2$s)",
194+
metadataView.nameComponent, propertyAction.value));
195+
}
196+
197+
Map<String, String> actualSqls =
198+
collectSqlStatements(overrideArguments.toArray(new String[] {}));
199+
200+
Assert.assertEquals(expectedSqls.size(), actualSqls.size());
201+
Assert.assertEquals(expectedSqls.keySet(), actualSqls.keySet());
202+
for (String name : expectedSqls.keySet()) {
203+
Assert.assertEquals(expectedSqls.get(name), actualSqls.get(name));
204+
}
205+
}
206+
}
207+
208+
@Test
209+
public void connector_assessment_generatesExpectedSql_withQueryOverrides() throws IOException {
210+
for (PropertyAction propertyAction : PropertyAction.values()) {
211+
TaskSqlMap expectedSqls =
212+
CoreMetadataDumpFormat.MAPPER.readValue(
213+
Resources.toString(
214+
Resources.getResource(
215+
String.format(
216+
"connector/snowflake/assessment-jdbc-tasks-sql-override-%s.yaml",
217+
propertyAction.value)),
218+
StandardCharsets.UTF_8),
219+
TaskSqlMap.class);
220+
221+
List<String> overrideArguments = new ArrayList<>();
222+
for (MetadataView metadataView : MetadataView.values()) {
223+
overrideArguments.add(
224+
String.format(
225+
"-Dsnowflake.metadata.%1$s.%2$s=SQL_OVERRIDE(%1$s, %2$s)",
226+
metadataView.nameComponent, propertyAction.value));
227+
}
228+
229+
overrideArguments.add("--assessment");
230+
Map<String, String> actualSqls =
231+
collectSqlStatements(overrideArguments.toArray(new String[] {}));
232+
233+
Assert.assertEquals(expectedSqls.size(), actualSqls.size());
234+
Assert.assertEquals(expectedSqls.keySet(), actualSqls.keySet());
235+
for (String name : expectedSqls.keySet()) {
236+
Assert.assertEquals(expectedSqls.get(name), actualSqls.get(name));
237+
}
238+
}
239+
}
240+
241+
private static Map<String, String> collectSqlStatements(String... extraArgs) throws IOException {
158242
List<Task<?>> tasks = new ArrayList<>();
159243
SnowflakeMetadataConnector connector = new SnowflakeMetadataConnector();
160-
connector.addTasksTo(tasks, new ConnectorArguments("--connector", connector.getName()));
244+
String[] args = ArrayUtils.addAll(new String[] {"--connector", connector.getName()}, extraArgs);
245+
connector.addTasksTo(tasks, new ConnectorArguments(args));
161246
return tasks.stream()
162247
.filter(t -> t instanceof JdbcSelectTask)
163248
.map(t -> (JdbcSelectTask) t)

‎dumper/app/src/test/resources/connector/snowflake/assessment-jdbc-tasks-sql-override-query.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
databases-au.csv: "SQL_OVERRIDE(databases, query)"
2+
schemata-au.csv: "SQL_OVERRIDE(schemata, query)"
3+
tables-au.csv: "SQL_OVERRIDE(tables, query)"
4+
columns-au.csv: "SQL_OVERRIDE(columns, query)"
5+
views-au.csv: "SQL_OVERRIDE(views, query)"
6+
functions-au.csv: "SQL_OVERRIDE(functions, query)"
7+
table_storage_metrics-au.csv: "SQL_OVERRIDE(storagemetrics, query)"
8+
warehouses.csv: "SHOW WAREHOUSES"
9+
external_tables.csv: "SHOW EXTERNAL TABLES"
10+
function_info.csv: "SHOW FUNCTIONS"

‎dumper/app/src/test/resources/connector/snowflake/assessment-jdbc-tasks-sql-override-where.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
databases-au.csv: "SELECT database_name, database_owner FROM SNOWFLAKE.ACCOUNT_USAGE.DATABASES WHERE SQL_OVERRIDE(databases, where)"
2+
schemata-au.csv: "SELECT catalog_name, schema_name FROM SNOWFLAKE.ACCOUNT_USAGE.SCHEMATA WHERE SQL_OVERRIDE(schemata, where)"
3+
tables-au.csv: "SELECT table_catalog, table_schema, table_name, table_type, row_count, bytes, clustering_key FROM SNOWFLAKE.ACCOUNT_USAGE.TABLES WHERE SQL_OVERRIDE(tables, where)"
4+
columns-au.csv: "SELECT table_catalog, table_schema, table_name, ordinal_position, column_name, data_type, is_nullable, column_default, character_maximum_length, numeric_precision, numeric_scale, datetime_precision, comment FROM SNOWFLAKE.ACCOUNT_USAGE.COLUMNS WHERE SQL_OVERRIDE(columns, where)"
5+
views-au.csv: "SELECT table_catalog, table_schema, table_name, view_definition FROM SNOWFLAKE.ACCOUNT_USAGE.VIEWS WHERE SQL_OVERRIDE(views, where)"
6+
functions-au.csv: "SELECT function_schema, function_name, data_type, argument_signature FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS WHERE SQL_OVERRIDE(functions, where)"
7+
table_storage_metrics-au.csv: "SELECT * FROM SNOWFLAKE.ACCOUNT_USAGE.TABLE_STORAGE_METRICS WHERE SQL_OVERRIDE(storagemetrics, where)"
8+
warehouses.csv: "SHOW WAREHOUSES"
9+
external_tables.csv: "SHOW EXTERNAL TABLES"
10+
function_info.csv: "SHOW FUNCTIONS"

‎dumper/app/src/test/resources/connector/snowflake/assessment-jdbc-tasks-sql.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
databases-au.csv: "SELECT database_name, database_owner FROM SNOWFLAKE.ACCOUNT_USAGE.DATABASES WHERE DELETED IS NULL"
2+
schemata-au.csv: "SELECT catalog_name, schema_name FROM SNOWFLAKE.ACCOUNT_USAGE.SCHEMATA WHERE DELETED IS NULL"
3+
tables-au.csv: "SELECT table_catalog, table_schema, table_name, table_type, row_count, bytes, clustering_key FROM SNOWFLAKE.ACCOUNT_USAGE.TABLES WHERE DELETED IS NULL"
4+
columns-au.csv: "SELECT table_catalog, table_schema, table_name, ordinal_position, column_name, data_type, is_nullable, column_default, character_maximum_length, numeric_precision, numeric_scale, datetime_precision, comment FROM SNOWFLAKE.ACCOUNT_USAGE.COLUMNS WHERE DELETED IS NULL"
5+
views-au.csv: "SELECT table_catalog, table_schema, table_name, view_definition FROM SNOWFLAKE.ACCOUNT_USAGE.VIEWS WHERE DELETED IS NULL"
6+
functions-au.csv: "SELECT function_schema, function_name, data_type, argument_signature FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS WHERE DELETED IS NULL"
7+
table_storage_metrics-au.csv: "SELECT * FROM SNOWFLAKE.ACCOUNT_USAGE.TABLE_STORAGE_METRICS"
8+
warehouses.csv: "SHOW WAREHOUSES"
9+
external_tables.csv: "SHOW EXTERNAL TABLES"
10+
function_info.csv: "SHOW FUNCTIONS"

‎dumper/app/src/test/resources/connector/snowflake/jdbc-tasks-sql-override-query.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
databases-au.csv: "SQL_OVERRIDE(databases, query)"
2+
databases.csv: "SQL_OVERRIDE(databases, query)"
3+
schemata-au.csv: "SQL_OVERRIDE(schemata, query)"
4+
schemata.csv: "SQL_OVERRIDE(schemata, query)"
5+
tables-au.csv: "SQL_OVERRIDE(tables, query)"
6+
tables.csv: "SQL_OVERRIDE(tables, query)"
7+
columns-au.csv: "SQL_OVERRIDE(columns, query)"
8+
columns.csv: "SQL_OVERRIDE(columns, query)"
9+
views-au.csv: "SQL_OVERRIDE(views, query)"
10+
views.csv: "SQL_OVERRIDE(views, query)"
11+
functions-au.csv: "SQL_OVERRIDE(functions, query)"
12+
functions.csv: "SQL_OVERRIDE(functions, query)"
13+
external_tables.csv: "SHOW EXTERNAL TABLES"

‎dumper/app/src/test/resources/connector/snowflake/jdbc-tasks-sql-override-where.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
databases-au.csv: "SELECT database_name, database_owner FROM SNOWFLAKE.ACCOUNT_USAGE.DATABASES WHERE SQL_OVERRIDE(databases, where)"
2+
databases.csv: "SELECT database_name, database_owner FROM INFORMATION_SCHEMA.DATABASES WHERE SQL_OVERRIDE(databases, where)"
3+
schemata-au.csv: "SELECT catalog_name, schema_name FROM SNOWFLAKE.ACCOUNT_USAGE.SCHEMATA WHERE SQL_OVERRIDE(schemata, where)"
4+
schemata.csv: "SELECT catalog_name, schema_name FROM INFORMATION_SCHEMA.SCHEMATA WHERE SQL_OVERRIDE(schemata, where)"
5+
tables-au.csv: "SELECT table_catalog, table_schema, table_name, table_type, row_count, bytes, clustering_key FROM SNOWFLAKE.ACCOUNT_USAGE.TABLES WHERE SQL_OVERRIDE(tables, where)"
6+
tables.csv: "SELECT table_catalog, table_schema, table_name, table_type, row_count, bytes, clustering_key FROM INFORMATION_SCHEMA.TABLES WHERE SQL_OVERRIDE(tables, where)"
7+
columns-au.csv: "SELECT table_catalog, table_schema, table_name, ordinal_position, column_name, data_type, is_nullable, column_default, character_maximum_length, numeric_precision, numeric_scale, datetime_precision, comment FROM SNOWFLAKE.ACCOUNT_USAGE.COLUMNS WHERE SQL_OVERRIDE(columns, where)"
8+
columns.csv: "SELECT table_catalog, table_schema, table_name, ordinal_position, column_name, data_type, is_nullable, column_default, character_maximum_length, numeric_precision, numeric_scale, datetime_precision, comment FROM INFORMATION_SCHEMA.COLUMNS WHERE SQL_OVERRIDE(columns, where)"
9+
views-au.csv: "SELECT table_catalog, table_schema, table_name, view_definition FROM SNOWFLAKE.ACCOUNT_USAGE.VIEWS WHERE SQL_OVERRIDE(views, where)"
10+
views.csv: "SELECT table_catalog, table_schema, table_name, view_definition FROM INFORMATION_SCHEMA.VIEWS WHERE SQL_OVERRIDE(views, where)"
11+
functions-au.csv: "SELECT function_schema, function_name, data_type, argument_signature FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS WHERE SQL_OVERRIDE(functions, where)"
12+
functions.csv: "SELECT function_schema, function_name, data_type, argument_signature FROM INFORMATION_SCHEMA.FUNCTIONS WHERE SQL_OVERRIDE(functions, where)"
13+
external_tables.csv: "SHOW EXTERNAL TABLES"

0 commit comments

Comments
 (0)