Skip to content

Commit 068a12a

Browse files
qifanliliqifanlili
authored
Merge branch 'apache:dev' into dev
2 parents be6840c + 879b1e2 commit 068a12a

File tree

66 files changed

+3391
-286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+3391
-286
lines changed

Diff for: ‎.github/workflows/labeler/label-scope-conf.yml

+5
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,11 @@ google-sheets:
143143
- changed-files:
144144
- any-glob-to-any-file: seatunnel-connectors-v2/connector-google-sheets/**
145145
- all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(google-sheets)/**'
146+
graphql:
147+
- all:
148+
- changed-files:
149+
- any-glob-to-any-file: seatunnel-connectors-v2/connector-graphql/**
150+
- all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(graphql)/**'
146151
hbase:
147152
- all:
148153
- changed-files:

Diff for: ‎config/plugin_config

+1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ connector-file-sftp
4545
connector-file-obs
4646
connector-google-sheets
4747
connector-google-firestore
48+
connector-graphql
4849
connector-hive
4950
connector-http-base
5051
connector-http-feishu

Diff for: ‎docs/en/connector-v2/changelog/connector-graphql.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<details><summary> Change Log </summary>
2+
3+
| Change | Commit | Version |
4+
| --- | --- | --- |
5+
|[Feature][Connector-V2] Add GraphQL source and sink (#7265)|https://github.com/apache/seatunnel/commit/dde6f9fcbd|2.3.9|
6+
7+
</details>

Diff for: ‎docs/en/connector-v2/sink/GraphQL.md

+179
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import ChangeLog from '../changelog/connector-graphql.md';
2+
3+
# GraphQL
4+
5+
> GraphQL sink connector
6+
7+
## Support Those Engines
8+
9+
> Spark<br/>
10+
> Flink<br/>
11+
> SeaTunnel Zeta<br/>
12+
13+
## Key Features
14+
15+
- [ ] [exactly-once](../../concept/connector-v2-features.md)
16+
- [ ] [cdc](../../concept/connector-v2-features.md)
17+
- [x] [support multiple table write](../../concept/connector-v2-features.md)
18+
19+
## Description
20+
21+
Used to launch web hooks using data.
22+
23+
> For example, if the data from upstream is [`label: {"__name__": "test1"}, value: 1.2.3,time:2024-08-15T17:00:00`], the body content is the following: `{"label":{"__name__": "test1"}, "value":"1.23","time":"2024-08-15T17:00:00"}`
24+
25+
**Tips: GraphQL sink only support `post json` webhook and the data from source will be treated as body content in web hook.And does not support passing past data**
26+
27+
## Supported DataSource Info
28+
29+
In order to use the Http connector, the following dependencies are required.
30+
They can be downloaded via install-plugin.sh or from the Maven central repository.
31+
32+
| Datasource | Supported Versions | Dependency |
33+
|------------|--------------------|------------------------------------------------------------------------------------------------------------------|
34+
| Http | universal | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-prometheus) |
35+
36+
## Sink Options
37+
38+
| Name | Type | Required | Default | Description |
39+
|-----------------------------|--------|----------|---------|-------------------------------------------------------------------------------------------------------------|
40+
| url | String | Yes | - | Http request url |
41+
| query | String | Yes | - | GraphQL query |
42+
| variables | String | No | - | GraphQL variables |
43+
| valueCover | Boolean | No | - | Whether the data overwrites the variable value |
44+
| headers | Map | No | - | Http headers |
45+
| retry | Int | No | - | The max retry times if request http return to `IOException` |
46+
| retry_backoff_multiplier_ms | Int | No | 100 | The retry-backoff times(millis) multiplier if request http failed |
47+
| retry_backoff_max_ms | Int | No | 10000 | The maximum retry-backoff times(millis) if request http failed |
48+
| connect_timeout_ms | Int | No | 12000 | Connection timeout setting, default 12s. |
49+
| socket_timeout_ms | Int | No | 60000 | Socket timeout setting, default 60s. |
50+
| key_timestamp | Int | NO | - | prometheus timestamp key . |
51+
| key_label | String | yes | - | prometheus label key |
52+
| key_value | Double | yes | - | prometheus value |
53+
| batch_size | Int | false | 1024 | prometheus batch size write |
54+
| flush_interval | Long | false | 300000L | prometheus flush commit interval |
55+
| common-options | | No | - | Sink plugin common parameters, please refer to [Sink Common Options](../sink-common-options.md) for details |
56+
57+
## Example
58+
59+
simple:
60+
61+
```hocon
62+
env {
63+
parallelism = 1
64+
job.mode = "BATCH"
65+
}
66+
67+
source {
68+
FakeSource {
69+
tables_configs = [
70+
{
71+
schema = {
72+
table = "graphql_sink_1"
73+
fields {
74+
id = int
75+
val_bool = boolean
76+
val_int8 = tinyint
77+
val_int16 = smallint
78+
val_int32 = int
79+
val_int64 = bigint
80+
val_float = float
81+
val_double = double
82+
val_decimal = "decimal(16, 1)"
83+
val_string = string
84+
val_unixtime_micros = timestamp
85+
}
86+
}
87+
rows = [
88+
{
89+
kind = INSERT
90+
fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "NEW", "2020-02-02T02:02:02"]
91+
}
92+
]
93+
},
94+
{
95+
schema = {
96+
table = "graphql_sink_2"
97+
fields {
98+
id = int
99+
val_bool = boolean
100+
val_int8 = tinyint
101+
val_int16 = smallint
102+
val_int32 = int
103+
val_int64 = bigint
104+
val_float = float
105+
val_double = double
106+
val_decimal = "decimal(16, 1)"
107+
val_string = string
108+
val_unixtime_micros = timestamp
109+
}
110+
}
111+
rows = [
112+
{
113+
kind = INSERT
114+
fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "NEW", "2020-02-02T02:02:02"]
115+
}
116+
]
117+
}
118+
]
119+
}
120+
}
121+
122+
sink {
123+
GraphQL {
124+
url = "http://192.168.1.103:9081/v1/graphql"
125+
query = """
126+
mutation MyMutation(
127+
$id: Int!
128+
$val_bool: Boolean!
129+
$val_int8: smallint!
130+
$val_int16: smallint!
131+
$val_int32: Int!
132+
$val_int64: bigint!
133+
$val_float: Float!
134+
$val_double: Float!
135+
$val_decimal: numeric!
136+
$val_string: String!
137+
$val_unixtime_micros: timestamp!
138+
) {
139+
insert_sink(objects: {
140+
id: $id,
141+
val_bool: $val_bool,
142+
val_int8: $val_int8,
143+
val_int16: $val_int16,
144+
val_int32: $val_int32,
145+
val_int64: $val_int64,
146+
val_float: $val_float,
147+
val_double: $val_double,
148+
val_decimal: $val_decimal,
149+
val_string: $val_string,
150+
val_unixtime_micros: $val_unixtime_micros
151+
}) {
152+
affected_rows
153+
returning {
154+
id
155+
val_bool
156+
val_decimal
157+
val_double
158+
val_float
159+
val_int16
160+
val_int32
161+
val_int64
162+
val_int8
163+
val_string
164+
val_unixtime_micros
165+
}
166+
}
167+
}
168+
"""
169+
variables = {
170+
"val_bool": True
171+
}
172+
}
173+
}
174+
175+
```
176+
177+
## Changelog
178+
179+
<ChangeLog />

Diff for: ‎docs/en/connector-v2/source/GraphQL.md

+188
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import ChangeLog from '../changelog/connector-graphql.md';
2+
3+
# GraphQL
4+
5+
> GraphQL source connector
6+
7+
## Description
8+
9+
Used to read data from GraphQL.
10+
11+
## Key features
12+
13+
- [x] [batch](../../concept/connector-v2-features.md)
14+
- [x] [stream](../../concept/connector-v2-features.md)
15+
- [ ] [parallelism](../../concept/connector-v2-features.md)
16+
17+
## Options
18+
19+
| name | type | required | default value |
20+
| --------------------------- | ------- | -------- | ----------------------- |
21+
| url | String | Yes | - |
22+
| query | String | Yes | - |
23+
| variables | Config | No | - |
24+
| enable_subscription | boolean | No | false |
25+
| timeout | Long | No | - |
26+
| content_field | String | Yes | $.data.{query_object}.* |
27+
| schema.fields | Config | Yes | - |
28+
| format | String | No | json |
29+
| params | Map | Yes | - |
30+
| poll_interval_millis | int | No | - |
31+
| retry | int | No | - |
32+
| retry_backoff_multiplier_ms | int | No | 100 |
33+
| retry_backoff_max_ms | int | No | 10000 |
34+
| enable_multi_lines | boolean | No | false |
35+
| common-options | config | No | - |
36+
37+
### url [String]
38+
39+
http request url
40+
41+
### query [String]
42+
43+
GraphQL expression query string
44+
45+
### variables [String]
46+
47+
GraphQL Variables
48+
49+
for example
50+
51+
```
52+
variables = {
53+
limit = 2
54+
}
55+
```
56+
57+
### enable_subscription [boolean]
58+
59+
1. true : Build a socket reader to subscribe to the GraphQL service
60+
2. false : Build an http reader subscription to the GraphQL service
61+
62+
### timeout [Long]
63+
64+
Time-out Period
65+
66+
### content_field [String]
67+
68+
JSONPath wildcard
69+
70+
### params [Map]
71+
72+
http request params
73+
74+
### poll_interval_millis [int]
75+
76+
request http api interval(millis) in stream mode
77+
78+
### retry [int]
79+
80+
The max retry times if request http return to `IOException`
81+
82+
### retry_backoff_multiplier_ms [int]
83+
84+
The retry-backoff times(millis) multiplier if request http failed
85+
86+
### retry_backoff_max_ms [int]
87+
88+
The maximum retry-backoff times(millis) if request http failed
89+
90+
### format [String]
91+
92+
the format of upstream data, default `json`.
93+
94+
### schema [Config]
95+
96+
Fill in a fixed value
97+
98+
```hocon
99+
schema = {
100+
fields {
101+
metric = "map<string, string>"
102+
value = double
103+
time = long
104+
}
105+
}
106+
107+
```
108+
109+
#### fields [Config]
110+
111+
the schema fields of upstream data
112+
113+
### common options
114+
115+
Source plugin common parameters, please refer to [Source Common Options](../source-common-options.md) for details
116+
117+
## Example
118+
119+
### Query
120+
121+
```hocon
122+
source {
123+
GraphQL {
124+
url = "http://192.168.1.103:9081/v1/graphql"
125+
format = "json"
126+
content_field = "$.data.source"
127+
query = """
128+
query MyQuery($limit: Int) {
129+
source(limit: $limit) {
130+
id
131+
val_bool
132+
val_double
133+
val_float
134+
}
135+
}
136+
"""
137+
variables = {
138+
limit = 2
139+
}
140+
schema = {
141+
fields {
142+
id = "int"
143+
val_bool = "boolean"
144+
val_double = "double"
145+
val_float = "float"
146+
}
147+
}
148+
}
149+
}
150+
```
151+
152+
### Subscription
153+
154+
```hocon
155+
source {
156+
GraphQL {
157+
url = "http://192.168.1.103:9081/v1/graphql"
158+
format = "json"
159+
content_field = "$.data.source"
160+
query = """
161+
query MyQuery($limit: Int) {
162+
source(limit: $limit) {
163+
id
164+
val_bool
165+
val_double
166+
val_float
167+
}
168+
}
169+
"""
170+
variables = {
171+
limit = 2
172+
}
173+
enable_subscription = true
174+
schema = {
175+
fields {
176+
id = "int"
177+
val_bool = "boolean"
178+
val_double = "double"
179+
val_float = "float"
180+
}
181+
}
182+
}
183+
}
184+
```
185+
186+
## Changelog
187+
188+
<ChangeLog />

0 commit comments

Comments
 (0)