|
27 | 27 | import org.apache.paimon.table.sink.ChannelComputer; |
28 | 28 | import org.apache.paimon.table.source.DataSplit; |
29 | 29 | import org.apache.paimon.table.source.EndOfScanException; |
| 30 | +import org.apache.paimon.table.source.IncrementalSplit; |
30 | 31 | import org.apache.paimon.table.source.SnapshotNotExistPlan; |
31 | 32 | import org.apache.paimon.table.source.StreamTableScan; |
32 | 33 | import org.apache.paimon.table.source.TableScan; |
@@ -305,20 +306,39 @@ protected synchronized void assignSplits() { |
305 | 306 | } |
306 | 307 |
|
307 | 308 | protected int assignSuggestedTask(FileStoreSourceSplit split) { |
308 | | - DataSplit dataSplit = ((DataSplit) split.split()); |
| 309 | + if (split.split() instanceof DataSplit) { |
| 310 | + return assignSuggestedTask((DataSplit) split.split()); |
| 311 | + } else { |
| 312 | + return assignSuggestedTask((IncrementalSplit) split.split()); |
| 313 | + } |
| 314 | + } |
| 315 | + |
| 316 | + protected int assignSuggestedTask(DataSplit split) { |
309 | 317 | int parallelism = context.currentParallelism(); |
310 | 318 |
|
311 | 319 | int bucketId; |
312 | | - if (dataSplit.bucket() == BucketMode.POSTPONE_BUCKET) { |
| 320 | + if (split.bucket() == BucketMode.POSTPONE_BUCKET) { |
313 | 321 | bucketId = |
314 | | - PostponeBucketFileStoreWrite.getWriteId(dataSplit.dataFiles().get(0).fileName()) |
| 322 | + PostponeBucketFileStoreWrite.getWriteId(split.dataFiles().get(0).fileName()) |
315 | 323 | % parallelism; |
316 | 324 | } else { |
317 | | - bucketId = dataSplit.bucket(); |
| 325 | + bucketId = split.bucket(); |
| 326 | + } |
| 327 | + |
| 328 | + if (shuffleBucketWithPartition) { |
| 329 | + return ChannelComputer.select(split.partition(), bucketId, parallelism); |
| 330 | + } else { |
| 331 | + return ChannelComputer.select(bucketId, parallelism); |
318 | 332 | } |
| 333 | + } |
| 334 | + |
| 335 | + protected int assignSuggestedTask(IncrementalSplit split) { |
| 336 | + int parallelism = context.currentParallelism(); |
319 | 337 |
|
| 338 | + // TODO how to deal with postpone bucket? |
| 339 | + int bucketId = split.bucket(); |
320 | 340 | if (shuffleBucketWithPartition) { |
321 | | - return ChannelComputer.select(dataSplit.partition(), bucketId, parallelism); |
| 341 | + return ChannelComputer.select(split.partition(), bucketId, parallelism); |
322 | 342 | } else { |
323 | 343 | return ChannelComputer.select(bucketId, parallelism); |
324 | 344 | } |
|
0 commit comments