Skip to content

Commit 91af24c

Browse files
priyabhatnagar25priyabhatnagar25
authored
Merge branch 'develop' into e2e_bigquerymultitable
2 parents 568fff4 + 00d2daa commit 91af24c

15 files changed

+941
-11
lines changed

‎pom.xml

-10
Original file line numberDiff line numberDiff line change
@@ -1217,16 +1217,6 @@
12171217
<version>1.2.8</version>
12181218
<scope>runtime</scope>
12191219
</dependency>
1220-
<dependency>
1221-
<groupId>com.google.apis</groupId>
1222-
<artifactId>google-api-services-storage</artifactId>
1223-
<version>v1-rev20220604-1.32.1</version>
1224-
</dependency>
1225-
<dependency>
1226-
<groupId>com.google.cloud</groupId>
1227-
<artifactId>google-cloud-storage</artifactId>
1228-
<version>2.8.0</version>
1229-
</dependency>
12301220
</dependencies>
12311221

12321222
</profile>

‎src/e2e-test/features/bigquery/sink/BigQueryToBigQueryAdditional.feature

+454
Large diffs are not rendered by default.

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
features = {"src/e2e-test/features"},
2828
glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
2929
"stepsdesign", "io.cdap.plugin.common.stepsdesign"},
30-
tags = {"@BigQuery_Sink"},
30+
tags = {"@BigQuery_Sink and not @CDAP-20830"},
31+
//TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
3132
monochrome = true,
3233
plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink",
3334
"json:target/cucumber-reports/cucumber-bigquery-sink.json",

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunnerRequired.java

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"stepsdesign", "io.cdap.plugin.common.stepsdesign"},
3030
tags = {"@BigQuery_Sink_Required"},
3131
monochrome = true,
32+
//TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
3233
plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink-required",
3334
"json:target/cucumber-reports/cucumber-bigquery-sink-required.json",
3435
"junit:target/cucumber-reports/cucumber-bigquery-sink-required.xml"}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

+256
Large diffs are not rendered by default.

‎src/e2e-test/resources/pluginParameters.properties

+25
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,31 @@ bqmtInvalidSinkReferenceName=(*^*&*
223223
bqmtInvalidTemporaryBucket=$#%$
224224
bqExpectedFile=testdata/BigQuery/BQExistingTableFile
225225
bgInsertDatatypeFile=testdata/BigQuery/BQInsertDatatypeFile
226+
TableKeyUpsert=ID
227+
TableKeyInsert=ID
228+
bqUpsertExpectedFile=testdata/BigQuery/BQUpsertTableFile
229+
bqUpdateDedupeExpectedFile=testdata/BigQuery/BQUpdateDedupeFile
230+
bqInsertExpectedFile=testdata/BigQuery/BQInsertIntFile
231+
relationTableKeyValue=Name
232+
dedupeByOrder=ASC
233+
dedupeByValue=ID
234+
dedupeByValueUpsert=Price
235+
rangeStartValue=2
236+
rangeEndValue=3
237+
rangeIntervalValue=1
238+
partitionByFieldValue=ID
239+
bqPartitionFieldDateTime=transaction_dt
240+
bqPartitionFieldTimeStamp=updated_on
241+
bqSourceTable2=dummy
242+
dedupeBy=DESC
243+
TableKeyDedupe=Name
244+
Directive_Drop=testdata/BigQuery/test_diffschema_record-cdap-data-pipeline.json
245+
bqUpsertDedupeFile=testdata/BigQuery/BQUpsertDedupeFile
246+
bqDifferentRecordFile=testdata/BigQuery/BQDifferentRecordNameFile
247+
bqDateExpectedFile=testdata/BigQuery/BQDateFile
248+
bqDateTimeExpectedFile=testdata/BigQuery/BQDateTimeFile
249+
bqTimeStampExpectedFile=testdata/BigQuery/BQTimeStampFile
250+
bqPartitionFieldDate=transaction_date
226251
## BIGQUERY-PLUGIN-PROPERTIES-END
227252

228253
## PUBSUBSINK-PLUGIN-PROPERTIES-START

‎src/e2e-test/resources/testdata/BigQuery/BQDateFile

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
2+
{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
3+
{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}

‎src/e2e-test/resources/testdata/BigQuery/BQDateTimeFile

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
2+
{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
3+
{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}

‎src/e2e-test/resources/testdata/BigQuery/BQDifferentRecordNameFile

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"ID":1,"Name":"string_1","Price":0.1}
2+
{"ID":1,"Name":"string_1","Price":0.1}

‎src/e2e-test/resources/testdata/BigQuery/BQInsertIntFile

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}
2+
{"Customer_Exists":true,"ID":3,"Name":"Rajan Kumar","Price":100.0}

‎src/e2e-test/resources/testdata/BigQuery/BQTimeStampFile

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"Firstname":"singh","ID":"Gurgaon","transaction_date":"2021-02-22","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
2+
{"Firstname":"Neera","ID":"Agra","transaction_date":"2021-02-20","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}
3+
{"Firstname":"","ID":"Noida","transaction_date":"2021-02-21","transaction_dt":"2019-07-07T11:24:00","updated_on":"2019-03-10T04:50:01Z"}

‎src/e2e-test/resources/testdata/BigQuery/BQUpdateDedupeFile

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"Customer_Exists":true,"ID":0,"Name":"string_0","Price":0.0}
2+
{"Customer_Exists":true,"ID":1,"Name":"string_1","Price":0.1}

‎src/e2e-test/resources/testdata/BigQuery/BQUpsertDedupeFile

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"Customer_Exists":false,"ID":3,"Name":"string_3","Price":0.3}
2+
{"Customer_Exists":false,"ID":2,"Name":"string_1","Price":0.2}
3+
{"Customer_Exists":true,"ID":0,"Name":"string_0","Price":0.0}

‎src/e2e-test/resources/testdata/BigQuery/BQUpsertTableFile

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"Customer_Exists":true,"ID":5,"Name":"Raja","Price":500.0}
2+
{"Customer_Exists":false,"ID":6,"Name":"Tom","Price":100.0}

‎src/e2e-test/resources/testdata/BigQuery/test_diffschema_record-cdap-data-pipeline.json

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
{
2+
"name": "test_diffschema_record",
3+
"description": "Data Pipeline Application",
4+
"artifact": {
5+
"name": "cdap-data-pipeline",
6+
"version": "6.10.0-SNAPSHOT",
7+
"scope": "SYSTEM"
8+
},
9+
"config": {
10+
"resources": {
11+
"memoryMB": 2048,
12+
"virtualCores": 1
13+
},
14+
"driverResources": {
15+
"memoryMB": 2048,
16+
"virtualCores": 1
17+
},
18+
"connections": [
19+
{
20+
"from": "BigQuery",
21+
"to": "Wrangler"
22+
},
23+
{
24+
"from": "Wrangler",
25+
"to": "BigQuery3"
26+
},
27+
{
28+
"from": "BigQuery2",
29+
"to": "BigQuery3"
30+
}
31+
],
32+
"comments": [],
33+
"postActions": [],
34+
"properties": {},
35+
"processTimingEnabled": true,
36+
"stageLoggingEnabled": false,
37+
"stages": [
38+
{
39+
"name": "BigQuery",
40+
"plugin": {
41+
"name": "BigQueryTable",
42+
"type": "batchsource",
43+
"label": "BigQuery",
44+
"artifact": {
45+
"name": "google-cloud",
46+
"version": "0.22.0-SNAPSHOT",
47+
"scope": "SYSTEM"
48+
},
49+
"properties": {
50+
"useConnection": "false",
51+
"project": "cdf-athena",
52+
"serviceAccountType": "filePath",
53+
"serviceFilePath": "auto-detect",
54+
"referenceName": "bq_ref",
55+
"dataset": "bq_automation",
56+
"table": "bqSourceTableMore",
57+
"enableQueryingViews": "false",
58+
"schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}"
59+
}
60+
},
61+
"outputSchema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}",
62+
"id": "BigQuery",
63+
"type": "batchsource",
64+
"label": "BigQuery",
65+
"icon": "fa-plug"
66+
},
67+
{
68+
"name": "Wrangler",
69+
"plugin": {
70+
"name": "Wrangler",
71+
"type": "transform",
72+
"label": "Wrangler",
73+
"artifact": {
74+
"name": "wrangler-transform",
75+
"version": "4.10.0-SNAPSHOT",
76+
"scope": "SYSTEM"
77+
},
78+
"properties": {
79+
"field": "*",
80+
"precondition": "false",
81+
"directives": "drop :TableName",
82+
"on-error": "fail-pipeline",
83+
"schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
84+
"workspaceId": "7038fc39-732e-4d75-8d3f-db6cfe5a11d8"
85+
}
86+
},
87+
"outputSchema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
88+
"inputSchema": [
89+
{
90+
"name": "BigQuery",
91+
"schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]},{\"name\":\"TableName\",\"type\":[\"string\",\"null\"]}]}"
92+
}
93+
],
94+
"id": "Wrangler",
95+
"type": "transform",
96+
"label": "Wrangler",
97+
"icon": "icon-DataPreparation"
98+
},
99+
{
100+
"name": "BigQuery3",
101+
"plugin": {
102+
"name": "BigQueryTable",
103+
"type": "batchsink",
104+
"label": "BigQuery3",
105+
"artifact": {
106+
"name": "google-cloud",
107+
"version": "0.22.0-SNAPSHOT",
108+
"scope": "SYSTEM"
109+
},
110+
"properties": {
111+
"useConnection": "false",
112+
"project": "auto-detect",
113+
"serviceAccountType": "filePath",
114+
"serviceFilePath": "auto-detect",
115+
"dataset": "bq_automation",
116+
"table": "New_target_table_combine",
117+
"operation": "insert",
118+
"truncateTable": "false",
119+
"allowSchemaRelaxation": "false",
120+
"location": "US",
121+
"createPartitionedTable": "false",
122+
"partitioningType": "TIME",
123+
"partitionFilterRequired": "false",
124+
"schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
125+
}
126+
},
127+
"outputSchema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
128+
"inputSchema": [
129+
{
130+
"name": "Wrangler",
131+
"schema": "{\"type\":\"record\",\"name\":\"record\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
132+
},
133+
{
134+
"name": "BigQuery2",
135+
"schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
136+
}
137+
],
138+
"id": "BigQuery3",
139+
"type": "batchsink",
140+
"label": "BigQuery3",
141+
"icon": "fa-plug"
142+
},
143+
{
144+
"name": "BigQuery2",
145+
"plugin": {
146+
"name": "BigQueryTable",
147+
"type": "batchsource",
148+
"label": "BigQuery2",
149+
"artifact": {
150+
"name": "google-cloud",
151+
"version": "0.22.0-SNAPSHOT",
152+
"scope": "SYSTEM"
153+
},
154+
"properties": {
155+
"useConnection": "false",
156+
"project": "cdf-athena",
157+
"serviceAccountType": "filePath",
158+
"serviceFilePath": "auto-detect",
159+
"referenceName": "bq_test",
160+
"dataset": "bq_automation",
161+
"table": "bqSourceTableLess",
162+
"enableQueryingViews": "false",
163+
"schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}"
164+
}
165+
},
166+
"outputSchema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"ID\",\"type\":[\"long\",\"null\"]},{\"name\":\"Name\",\"type\":[\"string\",\"null\"]},{\"name\":\"Price\",\"type\":[\"double\",\"null\"]}]}",
167+
"id": "BigQuery2",
168+
"type": "batchsource",
169+
"label": "BigQuery2",
170+
"icon": "fa-plug"
171+
}
172+
],
173+
"schedule": "0 1 */1 * *",
174+
"engine": "spark",
175+
"numOfRecordsPreview": 100,
176+
"rangeRecordsPreview": {
177+
"min": 1,
178+
"max": "5000"
179+
},
180+
"maxConcurrentRuns": 1
181+
},
182+
"version": "fe4ee1e3-6380-11ee-8217-0000003390c8"
183+
}

0 commit comments

Comments
 (0)