Skip to content

Commit 4e91070

Browse files
AnkitCLIAnkitCLI
authored
Merge pull request #544 from cloudsufi/e2e-tests_cloudsqlmysql
e2e cloudSqlMySql source additional scenarios
2 parents adfb81e + ad7156a commit 4e91070

File tree

7 files changed

+280
-1
lines changed

7 files changed

+280
-1
lines changed

‎cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlDesignTimeVaidation.feature renamed to ‎cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlDesignTimeValidation.feature

+64
Original file line numberDiff line numberDiff line change
@@ -180,3 +180,67 @@ Feature: CloudMySql source- Verify CloudMySql source plugin design time validati
180180
Then Click on the Validate button
181181
Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "errorMessageBoundingQuery"
182182
Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageNumberOfSplits"
183+
184+
@CloudMySql_Required
185+
Scenario: Verify CloudSQLMySQL source plugin validation errors for mandatory fields
186+
Given Open Datafusion Project to configure pipeline
187+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
188+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
189+
Then Click on the Validate button
190+
Then Verify mandatory property error for below listed properties:
191+
| jdbcPluginName |
192+
| connectionName |
193+
| database |
194+
| referenceName |
195+
| importQuery |
196+
197+
@CloudMySql_Required
198+
Scenario: To verify CloudSQLMySQL source plugin validation error message with invalid connection name with public instance
199+
Given Open Datafusion Project to configure pipeline
200+
When Expand Plugin group in the LHS plugins list: "Source"
201+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
202+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
203+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
204+
Then Select radio button plugin property: "instanceType" with value: "public"
205+
Then Replace input plugin property: "connectionName" with value: "invalidConnectionName"
206+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
207+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
208+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
209+
Then Replace input plugin property: "database" with value: "DatabaseName"
210+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
211+
Then Click on the Validate button
212+
Then Verify that the Plugin Property: "connectionName" is displaying an in-line error message: "errorMessageConnectionName"
213+
214+
@CloudMySql_Required
215+
Scenario: To verify CloudSQLMySQL source plugin validation error message with invalid connection name with private instance
216+
Given Open Datafusion Project to configure pipeline
217+
When Expand Plugin group in the LHS plugins list: "Source"
218+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
219+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
220+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
221+
Then Select radio button plugin property: "instanceType" with value: "private"
222+
Then Replace input plugin property: "connectionName" with value: "invalidConnectionName"
223+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
224+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
225+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
226+
Then Replace input plugin property: "database" with value: "DatabaseName"
227+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
228+
Then Click on the Validate button
229+
Then Verify that the Plugin Property: "connectionName" is displaying an in-line error message: "errorMessagePrivateConnectionName"
230+
231+
@CloudMySql_Required
232+
Scenario: To verify CloudSQLMySQL source plugin validation error message with invalid password
233+
Given Open Datafusion Project to configure pipeline
234+
When Expand Plugin group in the LHS plugins list: "Source"
235+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
236+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
237+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
238+
Then Select radio button plugin property: "instanceType" with value: "public"
239+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
240+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
241+
Then Replace input plugin property: "password" with value: "invalidPassword" for Credentials and Authorization related fields
242+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
243+
Then Replace input plugin property: "database" with value: "DatabaseName"
244+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
245+
Then Click on the Validate button
246+
Then Verify that the Plugin is displaying an error message: "errorMessageInvalidPassword" on the header

‎cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature

+89
Original file line numberDiff line numberDiff line change
@@ -230,3 +230,92 @@ Feature: CloudMySql Source - Run time scenarios
230230
Then Open Pipeline logs and verify Log entries having below listed Level and Message:
231231
| Level | Message |
232232
| ERROR | errorLogsMessageInvalidBoundingQuery |
233+
234+
@CLOUDSQLMYSQL_SOURCE_TEST @CLOUDSQLMYSQL_TARGET_TEST @CloudMySql_Required
235+
Scenario: To verify data is getting transferred from CloudMySql source to CloudMySql sink successfully with bounding query
236+
Given Open Datafusion Project to configure pipeline
237+
When Expand Plugin group in the LHS plugins list: "Source"
238+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
239+
When Expand Plugin group in the LHS plugins list: "Sink"
240+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Sink"
241+
Then Connect plugins: "CloudSQL MySQL" and "CloudSQL MySQL2" to establish connection
242+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
243+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
244+
Then Select radio button plugin property: "instanceType" with value: "public"
245+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
246+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
247+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
248+
Then Enter input plugin property: "referenceName" with value: "RefName"
249+
Then Enter input plugin property: "database" with value: "DatabaseName"
250+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
251+
Then Enter textarea plugin property: "boundingQuery" with value: "boundingQuery"
252+
Then Click on the Get Schema button
253+
Then Verify the Output Schema matches the Expected Schema: "outputSourceSchema"
254+
Then Validate "CloudSQL MySQL" plugin properties
255+
Then Close the Plugin Properties page
256+
Then Navigate to the properties page of plugin: "CloudSQL MySQL2"
257+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
258+
Then Select radio button plugin property: "instanceType" with value: "public"
259+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
260+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
261+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
262+
Then Enter input plugin property: "referenceName" with value: "RefName"
263+
Then Enter input plugin property: "database" with value: "DatabaseName"
264+
Then Replace input plugin property: "tableName" with value: "targetTable"
265+
Then Validate "CloudSQL MySQL2" plugin properties
266+
Then Close the Plugin Properties page
267+
Then Save the pipeline
268+
Then Preview and run the pipeline
269+
Then Verify the preview of pipeline is "success"
270+
Then Close the preview
271+
Then Deploy the pipeline
272+
Then Run the Pipeline in Runtime
273+
Then Wait till pipeline is in running state
274+
Then Open and capture logs
275+
Then Verify the pipeline status is "Succeeded"
276+
Then Validate the values of records transferred to target table is equal to the values from source table
277+
278+
@CLOUDSQLMYSQL_SOURCE_TEST @CLOUDSQLMYSQL_TARGET_TEST @CloudMySql_Required
279+
Scenario: To verify data is getting transferred from CloudMySql source to CloudMySql sink successfully when connection arguments are set to read boolean datatype
280+
Given Open Datafusion Project to configure pipeline
281+
When Expand Plugin group in the LHS plugins list: "Source"
282+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
283+
When Expand Plugin group in the LHS plugins list: "Sink"
284+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Sink"
285+
Then Connect plugins: "CloudSQL MySQL" and "CloudSQL MySQL2" to establish connection
286+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
287+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
288+
Then Select radio button plugin property: "instanceType" with value: "public"
289+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
290+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
291+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
292+
Then Enter input plugin property: "referenceName" with value: "RefName"
293+
Then Enter input plugin property: "database" with value: "DatabaseName"
294+
Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType"
295+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
296+
Then Click on the Get Schema button
297+
Then Verify the Output Schema matches the Expected Schema: "outputSourceSchema"
298+
Then Validate "CloudSQL MySQL" plugin properties
299+
Then Close the Plugin Properties page
300+
Then Navigate to the properties page of plugin: "CloudSQL MySQL2"
301+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
302+
Then Select radio button plugin property: "instanceType" with value: "public"
303+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
304+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
305+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
306+
Then Enter input plugin property: "referenceName" with value: "RefName"
307+
Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType"
308+
Then Enter input plugin property: "database" with value: "DatabaseName"
309+
Then Replace input plugin property: "tableName" with value: "targetTable"
310+
Then Validate "CloudSQL MySQL2" plugin properties
311+
Then Close the Plugin Properties page
312+
Then Save the pipeline
313+
Then Preview and run the pipeline
314+
Then Verify the preview of pipeline is "success"
315+
Then Close the preview
316+
Then Deploy the pipeline
317+
Then Run the Pipeline in Runtime
318+
Then Wait till pipeline is in running state
319+
Then Open and capture logs
320+
Then Verify the pipeline status is "Succeeded"
321+
Then Validate the values of records transferred to target table is equal to the values from source table

‎cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTimeMacro.feature

+56
Original file line numberDiff line numberDiff line change
@@ -305,3 +305,59 @@ Feature: CloudMySql - Verify CloudMySql plugin data transfer with macro argument
305305
Then Verify the pipeline status is "Succeeded"
306306
Then Close the pipeline logs
307307
Then Validate the values of records transferred to target Big Query table is equal to the values from source table
308+
309+
@CLOUDSQLMYSQL_SOURCE_TEST @CLOUDSQLMYSQL_TARGET_TEST @CloudMySql_Required
310+
Scenario: To verify data is getting transferred from CloudMySql to CloudMySql successfully when connection arguments,bounding query are macro enabled
311+
Given Open Datafusion Project to configure pipeline
312+
When Expand Plugin group in the LHS plugins list: "Source"
313+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Source"
314+
When Expand Plugin group in the LHS plugins list: "Sink"
315+
When Select plugin: "CloudSQL MySQL" from the plugins list as: "Sink"
316+
Then Connect plugins: "CloudSQL MySQL" and "CloudSQL MySQL2" to establish connection
317+
Then Navigate to the properties page of plugin: "CloudSQL MySQL"
318+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
319+
Then Select radio button plugin property: "instanceType" with value: "public"
320+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
321+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
322+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
323+
Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSource"
324+
Then Enter input plugin property: "referenceName" with value: "RefName"
325+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
326+
Then Enter input plugin property: "database" with value: "DatabaseName"
327+
Then Click on the Macro button of Property: "boundingQuery" and set the value in textarea: "CloudMysqlBoundingQuery"
328+
Then Validate "CloudSQL MySQL" plugin properties
329+
Then Close the Plugin Properties page
330+
Then Navigate to the properties page of plugin: "CloudSQL MySQL2"
331+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
332+
Then Select radio button plugin property: "instanceType" with value: "public"
333+
Then Replace input plugin property: "connectionName" with value: "connectionName" for Credentials and Authorization related fields
334+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
335+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
336+
Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink"
337+
Then Enter input plugin property: "referenceName" with value: "RefName"
338+
Then Enter input plugin property: "database" with value: "DatabaseName"
339+
Then Replace input plugin property: "tableName" with value: "targetTable"
340+
Then Validate "CloudSQL MySQL2" plugin properties
341+
Then Close the Plugin Properties page
342+
Then Save the pipeline
343+
Then Preview and run the pipeline
344+
Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource"
345+
Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink"
346+
Then Enter runtime argument value "boundingQuery" for key "CloudMysqlBoundingQuery"
347+
Then Run the preview of pipeline with runtime arguments
348+
Then Wait till pipeline preview is in running state
349+
Then Open and capture pipeline preview logs
350+
Then Verify the preview run status of pipeline in the logs is "succeeded"
351+
Then Close the pipeline logs
352+
Then Close the preview
353+
Then Deploy the pipeline
354+
Then Run the Pipeline in Runtime
355+
Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource"
356+
Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink"
357+
Then Enter runtime argument value "boundingQuery" for key "CloudMysqlBoundingQuery"
358+
Then Run the Pipeline in Runtime with runtime arguments
359+
Then Wait till pipeline is in running state
360+
Then Open and capture logs
361+
Then Verify the pipeline status is "Succeeded"
362+
Then Close the pipeline logs
363+
Then Validate the values of records transferred to target table is equal to the values from source table

‎cloudsql-mysql-plugin/src/e2e-test/java/io/cdap/plugin/CloudMySqlClient.java

+33
Original file line numberDiff line numberDiff line change
@@ -178,4 +178,37 @@ public static void deleteTable(String table)
178178
statement.execute(dropTableQuery);
179179
}
180180
}
181+
182+
public static void createSourceTestTable(String sourceTable) throws SQLException, ClassNotFoundException {
183+
try (Connection connect = getCloudSqlConnection();
184+
Statement statement = connect.createStatement()) {
185+
String createSourceTableQuery = "CREATE TABLE IF NOT EXISTS " + sourceTable +
186+
"(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)";
187+
statement.executeUpdate(createSourceTableQuery);
188+
189+
// Truncate table to clean the data of last failure run.
190+
String truncateSourceTableQuery = "TRUNCATE TABLE " + sourceTable;
191+
statement.executeUpdate(truncateSourceTableQuery);
192+
193+
// Insert dummy data.
194+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" +
195+
"VALUES (1, 'Simpson', true)");
196+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" +
197+
"VALUES (2, 'McBeal', true)");
198+
statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" +
199+
"VALUES (3, 'Flinstone', false)");
200+
}
201+
}
202+
203+
public static void createTargetTestTable(String targetTable) throws SQLException, ClassNotFoundException {
204+
try (Connection connect = getCloudSqlConnection();
205+
Statement statement = connect.createStatement()) {
206+
String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable +
207+
"(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)";
208+
statement.executeUpdate(createTargetTableQuery);
209+
// Truncate table to clean the data of last failure run.
210+
String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable;
211+
statement.executeUpdate(truncateTargetTableQuery);
212+
}
213+
}
181214
}

0 commit comments

Comments
 (0)