Skip to content

Commit d4cd1dc

Browse files
AnkitCLIAnkitCLI
committed
e2e tests PostgreSql Sink
1 parent 4e91070 commit d4cd1dc

File tree

4 files changed

+172
-0
lines changed

4 files changed

+172
-0
lines changed

‎postgresql-plugin/src/e2e-test/features/postgresql/sink/PostgresqlRunTime.feature

+98
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,101 @@ Feature: PostgreSQL - Verify data transfer from BigQuery source to PostgreSQL si
144144
Then Open and capture logs
145145
Then Verify the pipeline status is "Succeeded"
146146
Then Validate the values of records transferred to target PostgreSQL table is equal to the values from source BigQuery table
147+
148+
@BQ_SOURCE_TEST @Postgresql_Required @POSTGRESQL_TEST_TABLE @Plugin-1526
149+
Scenario Outline: To verify data is getting transferred from BigQuery source to PostgreSQL sink with Advanced operations Upsert and Update for table key
150+
Given Open Datafusion Project to configure pipeline
151+
When Expand Plugin group in the LHS plugins list: "Source"
152+
When Select plugin: "BigQuery" from the plugins list as: "Source"
153+
When Expand Plugin group in the LHS plugins list: "Sink"
154+
When Select plugin: "PostgreSQL" from the plugins list as: "Sink"
155+
Then Connect plugins: "BigQuery" and "PostgreSQL" to establish connection
156+
Then Navigate to the properties page of plugin: "BigQuery"
157+
Then Replace input plugin property: "project" with value: "projectId"
158+
Then Enter input plugin property: "datasetProject" with value: "projectId"
159+
Then Enter input plugin property: "referenceName" with value: "BQReferenceName"
160+
Then Enter input plugin property: "dataset" with value: "dataset"
161+
Then Enter input plugin property: "table" with value: "bqSourceTable"
162+
Then Click on the Get Schema button
163+
Then Verify the Output Schema matches the Expected Schema: "bqOutputMultipleDatatypesSchema"
164+
Then Validate "BigQuery" plugin properties
165+
Then Close the Plugin Properties page
166+
Then Navigate to the properties page of plugin: "PostgreSQL"
167+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
168+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
169+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
170+
Then Replace input plugin property: "database" with value: "databaseName"
171+
Then Replace input plugin property: "tableName" with value: "targetTable"
172+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
173+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
174+
Then Enter input plugin property: "referenceName" with value: "targetRef"
175+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
176+
Then Select radio button plugin property: "opeationName" with value: "<options>"
177+
Then Click on the Add Button of the property: "relationTableKey" with value:
178+
| PostgreSQLTableKey |
179+
Then Validate "PostgreSQL" plugin properties
180+
Then Close the Plugin Properties page
181+
Then Save the pipeline
182+
Then Preview and run the pipeline
183+
Then Verify the preview of pipeline is "success"
184+
Then Click on preview data for PostgreSQL sink
185+
Then Close the preview data
186+
Then Deploy the pipeline
187+
Then Run the Pipeline in Runtime
188+
Then Wait till pipeline is in running state
189+
Then Open and capture logs
190+
Then Verify the pipeline status is "Succeeded"
191+
Then Validate the values of records transferred to target PostgreSQL table is equal to the values from source BigQuery table
192+
Examples:
193+
| UPSERT |
194+
| UPDATE |
195+
196+
@BQ_SOURCE_TEST @Postgresql_Required @POSTGRESQL_TEST_TABLE @CONNECTION @Plugin-1526
197+
Scenario: To verify data is getting transferred from BigQuery source to PostgreSQL sink successfully using Connection
198+
Given Open Datafusion Project to configure pipeline
199+
When Expand Plugin group in the LHS plugins list: "Source"
200+
When Select plugin: "BigQuery" from the plugins list as: "Source"
201+
When Expand Plugin group in the LHS plugins list: "Sink"
202+
When Select plugin: "PostgreSQL" from the plugins list as: "Sink"
203+
Then Connect plugins: "BigQuery" and "PostgreSQL" to establish connection
204+
Then Navigate to the properties page of plugin: "BigQuery"
205+
Then Replace input plugin property: "project" with value: "projectId"
206+
Then Enter input plugin property: "datasetProject" with value: "projectId"
207+
Then Enter input plugin property: "referenceName" with value: "BQReferenceName"
208+
Then Enter input plugin property: "dataset" with value: "dataset"
209+
Then Enter input plugin property: "table" with value: "bqSourceTable"
210+
Then Click on the Get Schema button
211+
Then Validate "BigQuery" plugin properties
212+
Then Close the Plugin Properties page
213+
Then Navigate to the properties page of plugin: "PostgreSQL"
214+
And Click plugin property: "switch-useConnection"
215+
And Click on the Browse Connections button
216+
And Click on the Add Connection button
217+
Then Click plugin property: "connector-PostgreSQL"
218+
And Enter input plugin property: "name" with value: "connection.name"
219+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
220+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
221+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
222+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
223+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
224+
Then Replace input plugin property: "database" with value: "databaseName"
225+
Then Click on the Test Connection button
226+
And Verify the test connection is successful
227+
Then Click on the Create button
228+
Then Select connection: "connection.name"
229+
Then Enter input plugin property: "referenceName" with value: "targetRef"
230+
Then Replace input plugin property: "tableName" with value: "targetTable"
231+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
232+
Then Validate "PostgreSQL" plugin properties
233+
Then Close the Plugin Properties page
234+
Then Save the pipeline
235+
Then Preview and run the pipeline
236+
Then Verify the preview of pipeline is "success"
237+
Then Click on preview data for PostgreSQL sink
238+
Then Close the preview data
239+
Then Deploy the pipeline
240+
Then Run the Pipeline in Runtime
241+
Then Wait till pipeline is in running state
242+
Then Open and capture logs
243+
Then Verify the pipeline status is "Succeeded"
244+
Then Validate the values of records transferred to target PostgreSQL table is equal to the values from source BigQuery table

‎postgresql-plugin/src/e2e-test/features/postgresql/sink/PostgresqlRunTimeMacro.feature

+48
Original file line numberDiff line numberDiff line change
@@ -136,3 +136,51 @@ Feature: PostgreSQL - Verify data transfer to PostgreSQL sink with macro argumen
136136
Then Verify the pipeline status is "Succeeded"
137137
Then Close the pipeline logs
138138
Then Validate the values of records transferred to target PostgreSQL table is equal to the values from source BigQuery table
139+
140+
@BQ_SOURCE_TEST @Postgresql_Required @POSTGRESQL_TEST_TABLE @PLUGIN-1628 @Plugin-1526
141+
Scenario: To verify data is getting transferred from BigQuery source to PostgreSQL sink using connection arguments and operations as macro
142+
Given Open Datafusion Project to configure pipeline
143+
When Expand Plugin group in the LHS plugins list: "Source"
144+
When Select plugin: "BigQuery" from the plugins list as: "Source"
145+
When Expand Plugin group in the LHS plugins list: "Sink"
146+
When Select plugin: "PostgreSQL" from the plugins list as: "Sink"
147+
Then Connect plugins: "BigQuery" and "PostgreSQL" to establish connection
148+
Then Navigate to the properties page of plugin: "BigQuery"
149+
Then Enter input plugin property: "referenceName" with value: "BQReferenceName"
150+
Then Click on the Macro button of Property: "projectId" and set the value to: "bqProjectId"
151+
Then Click on the Macro button of Property: "datasetProjectId" and set the value to: "bqDatasetProjectId"
152+
Then Click on the Macro button of Property: "dataset" and set the value to: "bqDataset"
153+
Then Click on the Macro button of Property: "table" and set the value to: "bqTable"
154+
Then Validate "BigQuery" plugin properties
155+
Then Close the Plugin Properties page
156+
Then Navigate to the properties page of plugin: "PostgreSQL"
157+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
158+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
159+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
160+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
161+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
162+
Then Enter input plugin property: "referenceName" with value: "targetRef"
163+
Then Replace input plugin property: "database" with value: "databaseName"
164+
Then Click on the Macro button of Property: "connectionArguments" and set the value to: "PostgreSQLConnectionArguments"
165+
Then Click on the Macro button of Property: "operationName" and set the value to: "PostgreSQLOperationName"
166+
Then Click on the Macro button of Property: "tableName" and set the value to: "PostgreSQLTableName"
167+
Then Click on the Macro button of Property: "dbSchemaName" and set the value to: "PostgreSQLSchemaName"
168+
Then Validate "PostgreSQL" plugin properties
169+
Then Close the Plugin Properties page
170+
Then Save the pipeline
171+
Then Deploy the pipeline
172+
Then Run the Pipeline in Runtime
173+
Then Enter runtime argument value "projectId" for key "bqProjectId"
174+
Then Enter runtime argument value "projectId" for key "bqDatasetProjectId"
175+
Then Enter runtime argument value "dataset" for key "bqDataset"
176+
Then Enter runtime argument value "bqSourceTable" for key "bqTable"
177+
Then Enter runtime argument value "PostgreSQLConnectionArgumentsList" for key "PostgreSQLConnectionArgumentsList"
178+
Then Enter runtime argument value "PostgreSQLOperationName" for key "PostgreSQLOperationName"
179+
Then Enter runtime argument value "targetTable" for key "PostgreSQLTableName"
180+
Then Enter runtime argument value "schema" for key "PostgreSQLSchemaName"
181+
Then Run the Pipeline in Runtime with runtime arguments
182+
Then Wait till pipeline is in running state
183+
Then Open and capture logs
184+
Then Verify the pipeline status is "Succeeded"
185+
Then Close the pipeline logs
186+
Then Validate the values of records transferred to target PostgreSQL table is equal to the values from source BigQuery table

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

+23
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package io.cdap.plugin.common.stepsdesign;
1818

1919
import com.google.cloud.bigquery.BigQueryException;
20+
import io.cdap.e2e.pages.actions.CdfConnectionActions;
21+
import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions;
2022
import io.cdap.e2e.utils.BigQueryClient;
2123
import io.cdap.e2e.utils.PluginPropertyUtils;
2224
import io.cdap.plugin.PostgresqlClient;
@@ -159,4 +161,25 @@ private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile
159161
PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable);
160162
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " created successfully");
161163
}
164+
165+
@Before(order = 1, value = "@CONNECTION")
166+
public static void setNewConnectionName() {
167+
String connectionName = "PostgreSql" + RandomStringUtils.randomAlphanumeric(10);
168+
PluginPropertyUtils.addPluginProp("connection.name", connectionName);
169+
BeforeActions.scenario.write("New Connection name: " + connectionName);
170+
}
171+
172+
private static void deleteConnection(String connectionType, String connectionName) throws IOException {
173+
CdfConnectionActions.openWranglerConnectionsPage();
174+
CdfConnectionActions.expandConnections(connectionType);
175+
CdfConnectionActions.openConnectionActionMenu(connectionType, connectionName);
176+
CdfConnectionActions.selectConnectionAction(connectionType, connectionName, "Delete");
177+
CdfPluginPropertiesActions.clickPluginPropertyButton("Delete");
178+
}
179+
180+
@After(order = 1, value = "@CONNECTION")
181+
public static void deleteBQConnection() throws IOException {
182+
deleteConnection("PostgreSql", "connection.name");
183+
PluginPropertyUtils.removePluginProp("connection.name");
184+
}
162185
}

‎postgresql-plugin/src/e2e-test/resources/pluginParameters.properties

+3
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ invalidBoundingQuery=SELECT MIN(id),MAX(id) FROM table
7272
invalidBoundingQueryValue=select;
7373
invalidTable=table
7474
#POSTGRESQL Valid Properties
75+
PostgreSQLConnectionArgumentsList=fetchsize=1000
76+
PostgreSQLOperationName=INSERT
77+
PostgreSQLTableKey=col2
7578
connectionArgumentsList=[{"key":"queryTimeout","value":"-1"}]
7679
connectionTimeout=150
7780
numberOfSplits=2

0 commit comments

Comments
 (0)