Skip to content

Commit 568fab0

Browse files
committed
fix: fix some bugs and add docs
1 parent becc7be commit 568fab0

5 files changed

Lines changed: 116 additions & 3 deletions

File tree

docs/en/transforms/embedding.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,57 @@ vectorization_fields {
105105
}
106106
```
107107

108+
**Multi-field Mixing Multimodal Vectorization:**
109+
```hocon
110+
vectorization_fields {
111+
# Multi-field text
112+
multi_field_text_vector = [product_name, description]
113+
114+
# Multi-field image
115+
multi_field_image_vector = [
116+
{
117+
field = product_image_url
118+
modality = jpeg
119+
format = url
120+
},
121+
{
122+
field = thumbnail_image
123+
modality = png
124+
format = url
125+
}
126+
]
127+
128+
# Multi-field video
129+
multi_field_video_vector = [
130+
{
131+
field = product_video_url
132+
modality = mp4
133+
format = url
134+
},
135+
{
136+
field = promotional_video
137+
modality = mov
138+
format = url
139+
}
140+
]
141+
142+
# Multi-field mix multimodal
143+
multi_field_mix_vector = [
144+
product_name,
145+
{
146+
field = product_image_url
147+
modality = jpeg
148+
format = url
149+
},
150+
{
151+
field = product_video_url
152+
modality = mp4
153+
format = url
154+
}
155+
]
156+
}
157+
```
158+
108159
**Field Specification Formats:**
109160

110161
**Supported Modality Types:**

docs/zh/transforms/embedding.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,57 @@ vectorization_fields {
9898
}
9999
```
100100

101+
**多字段混合多模态向量化:**
102+
```hocon
103+
vectorization_fields {
104+
# 多字段文本
105+
multi_field_text_vector = [product_name, description]
106+
107+
# 多字段图片
108+
multi_field_image_vector = [
109+
{
110+
field = product_image_url
111+
modality = jpeg
112+
format = url
113+
},
114+
{
115+
field = thumbnail_image
116+
modality = png
117+
format = url
118+
}
119+
]
120+
121+
# 多字段视频
122+
multi_field_video_vector = [
123+
{
124+
field = product_video_url
125+
modality = mp4
126+
format = url
127+
},
128+
{
129+
field = promotional_video
130+
modality = mov
131+
format = url
132+
}
133+
]
134+
135+
# 多字段混合多模态
136+
multi_field_mix_vector = [
137+
product_name,
138+
{
139+
field = product_image_url
140+
modality = jpeg
141+
format = url
142+
},
143+
{
144+
field = product_video_url
145+
modality = mp4
146+
format = url
147+
}
148+
]
149+
}
150+
```
151+
101152
**字段规范格式:**
102153

103154
**支持的模态类型:**

seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
import java.net.URISyntaxException;
5252
import java.nio.ByteBuffer;
5353
import java.util.ArrayList;
54-
import java.util.HashMap;
54+
import java.util.LinkedHashMap;
5555
import java.util.List;
5656
import java.util.Map;
5757
import java.util.Set;
@@ -204,7 +204,7 @@ private void initOutputFields(SeaTunnelRowType inputRowType, ReadonlyConfig conf
204204
}
205205

206206
List<String> fieldNames = new ArrayList<>();
207-
Map<VectorFieldSpec, List<Integer>> fieldSpecMap = new HashMap<>();
207+
Map<VectorFieldSpec, List<Integer>> fieldSpecMap = new LinkedHashMap<>();
208208
for (Map.Entry<String, Object> fieldConfig : fieldsConfig.entrySet()) {
209209
VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(fieldConfig);
210210
log.info("Vector field spec: {}", vectorFieldSpec);

seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/SrcField.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ public String toBase64() {
4343
if (fieldValue == null) {
4444
throw new IllegalArgumentException("Binary data cannot be null or empty");
4545
}
46-
return Base64.getEncoder().encodeToString(fieldValue.toString().getBytes());
46+
if (fieldValue instanceof byte[]) {
47+
return Base64.getEncoder().encodeToString((byte[]) fieldValue);
48+
} else {
49+
return Base64.getEncoder().encodeToString(fieldValue.toString().getBytes());
50+
}
4751
}
4852
}

seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
import java.io.IOException;
3434
import java.util.Arrays;
35+
import java.util.Base64;
3536
import java.util.Collections;
3637
import java.util.HashMap;
3738
import java.util.List;
@@ -218,6 +219,12 @@ void testMultimodalBodyWithBinaryImage() {
218219
ObjectNode inputNode = (ObjectNode) result.get("input").get(0);
219220
Assertions.assertEquals("image_url", inputNode.get("type").asText());
220221
Assertions.assertTrue(inputNode.has("image_url"));
222+
Assertions.assertTrue(
223+
inputNode
224+
.get("image_url")
225+
.get("url")
226+
.asText()
227+
.endsWith(Base64.getEncoder().encodeToString(mockImageData)));
221228
}
222229

223230
/**

0 commit comments

Comments
 (0)