[Improve][Zeta] Support restore execute savemode (#9059)

hailin0 · web-flow · commit 5e0e376a90c3 · 2025-04-08T13:32:34.000+08:00
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSaveModeHandler.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSaveModeHandler.java
@@ -117,6 +117,16 @@ public void handleDataSaveMode() {
         }
     }
 
+    @Override
+    public void handleSchemaSaveModeWithRestore() {
+        if (SchemaSaveMode.ERROR_WHEN_SCHEMA_NOT_EXIST == schemaSaveMode) {
+            errorWhenSchemaNotExist();
+        } else if (SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST == schemaSaveMode
+                || SchemaSaveMode.RECREATE_SCHEMA == schemaSaveMode) {
+            createSchemaWhenNotExist();
+        }
+    }
+
     protected void recreateSchema() {
         if (tableExists()) {
             dropTable();
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SaveModeHandler.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SaveModeHandler.java
@@ -28,6 +28,8 @@ public interface SaveModeHandler extends AutoCloseable {
 
     void handleDataSaveMode();
 
+    void handleSchemaSaveModeWithRestore();
+
     SchemaSaveMode getSchemaSaveMode();
 
     DataSaveMode getDataSaveMode();
diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/DefaultSaveModeHandlerTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/DefaultSaveModeHandlerTest.java
@@ -22,15 +22,24 @@
 import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
 import org.apache.seatunnel.api.table.catalog.InMemoryCatalog;
 import org.apache.seatunnel.api.table.catalog.InMemoryCatalogFactory;
+import org.apache.seatunnel.api.table.catalog.TablePath;
 import org.apache.seatunnel.api.table.type.BasicType;
 import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
 import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class DefaultSaveModeHandlerTest {
 
@@ -115,6 +124,61 @@ public void shouldNotTruncateRecreatedTable() {
                 "Should not truncate data for recreated table");
     }
 
+    @Test
+    public void handlesErrorWhenSchemaNotExist() {
+        Catalog catalog = mock(Catalog.class);
+        CatalogTable catalogTable = createCatalogTable("notExistsTable");
+        when(catalog.tableExists(any(TablePath.class))).thenReturn(false);
+        DefaultSaveModeHandler handler =
+                new DefaultSaveModeHandler(
+                        SchemaSaveMode.ERROR_WHEN_SCHEMA_NOT_EXIST,
+                        DataSaveMode.APPEND_DATA,
+                        catalog,
+                        catalogTable,
+                        null);
+
+        assertThrows(SeaTunnelRuntimeException.class, handler::handleSchemaSaveModeWithRestore);
+    }
+
+    @Test
+    public void createsSchemaWhenNotExist() {
+        CatalogTable catalogTable = createCatalogTable("notExistsTable");
+
+        Catalog catalog = mock(Catalog.class);
+        when(catalog.tableExists(any(TablePath.class))).thenReturn(false);
+        DefaultSaveModeHandler handler =
+                new DefaultSaveModeHandler(
+                        SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST,
+                        DataSaveMode.APPEND_DATA,
+                        catalog,
+                        catalogTable,
+                        null);
+
+        handler.handleSchemaSaveModeWithRestore();
+
+        verify(catalog, times(1))
+                .createTable(any(TablePath.class), any(CatalogTable.class), eq(true));
+    }
+
+    @Test
+    public void recreatesSchemaWhenNotExist() {
+        CatalogTable catalogTable = createCatalogTable("notExistsTable");
+        Catalog catalog = mock(Catalog.class);
+        when(catalog.tableExists(any(TablePath.class))).thenReturn(false);
+        DefaultSaveModeHandler handler =
+                new DefaultSaveModeHandler(
+                        SchemaSaveMode.RECREATE_SCHEMA,
+                        DataSaveMode.APPEND_DATA,
+                        catalog,
+                        catalogTable,
+                        null);
+
+        handler.handleSchemaSaveModeWithRestore();
+
+        verify(catalog, times(1))
+                .createTable(any(TablePath.class), any(CatalogTable.class), eq(true));
+    }
+
     private CatalogTable createCatalogTable(String tableName) {
         return CatalogTableUtil.getCatalogTable("", "st", "public", tableName, rowType);
     }
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/sink/inmemory/InMemorySaveModeHandler.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/sink/inmemory/InMemorySaveModeHandler.java
@@ -55,6 +55,9 @@ public void handleDataSaveMode() {
         log.info("handle data savemode with table path: {}", catalogTable.getTablePath());
     }
 
+    @Override
+    public void handleSchemaSaveModeWithRestore() {}
+
     @Override
     public SchemaSaveMode getSchemaSaveMode() {
         return SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST;
diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java
@@ -690,6 +690,8 @@ private static <T> T findLast(LinkedHashMap<?, T> map) {
                         actionConfig);
         if (!isStartWithSavePoint) {
             handleSaveMode(sink);
+        } else {
+            handleSchemaSaveModeWithRestore(sink);
         }
         sinkAction.setParallelism(parallelism);
         return sinkAction;
@@ -716,6 +718,25 @@ public void handleSaveMode(SeaTunnelSink<?, ?, ?, ?> sink) {
         }
     }
 
+    public void handleSchemaSaveModeWithRestore(SeaTunnelSink<?, ?, ?, ?> sink) {
+        if (SupportSaveMode.class.isAssignableFrom(sink.getClass())) {
+            SupportSaveMode saveModeSink = (SupportSaveMode) sink;
+            if (envOptions
+                    .get(EnvCommonOptions.SAVEMODE_EXECUTE_LOCATION)
+                    .equals(SaveModeExecuteLocation.CLIENT)) {
+                Optional<SaveModeHandler> saveModeHandler = saveModeSink.getSaveModeHandler();
+                if (saveModeHandler.isPresent()) {
+                    try (SaveModeHandler handler = saveModeHandler.get()) {
+                        handler.open();
+                        handler.handleSchemaSaveModeWithRestore();
+                    } catch (Exception e) {
+                        throw new SeaTunnelRuntimeException(HANDLE_SAVE_MODE_FAILED, e);
+                    }
+                }
+            }
+        }
+    }
+
     private List<URL> getSourcePluginJarPaths(Config sourceConfig) {
         SeaTunnelSourcePluginDiscovery sourcePluginDiscovery = new SeaTunnelSourcePluginDiscovery();
         PluginIdentifier pluginIdentifier =
diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
@@ -271,7 +271,8 @@ public synchronized void init(long initializationTimestamp, boolean restart) thr
                                                     logicalVertexIdClassLoaderMap.get(
                                                             sink.getId()));
                                     JobMaster.handleSaveMode(
-                                            ((SinkAction<?, ?, ?, ?>) sink).getSink());
+                                            ((SinkAction<?, ?, ?, ?>) sink).getSink(),
+                                            logicalDag.isStartWithSavePoint());
                                 });
                 Thread.currentThread().setContextClassLoader(appClassLoader);
             }
@@ -556,22 +557,26 @@ public void run() {
         }
     }
 
-    public static void handleSaveMode(SeaTunnelSink sink) {
+    public static void handleSaveMode(SeaTunnelSink sink, boolean isStartWithSavePoint) {
         if (sink instanceof SupportSaveMode) {
             Optional<SaveModeHandler> saveModeHandler =
                     ((SupportSaveMode) sink).getSaveModeHandler();
             if (saveModeHandler.isPresent()) {
                 try (SaveModeHandler handler = saveModeHandler.get()) {
                     handler.open();
-                    new SaveModeExecuteWrapper(handler).execute();
+                    if (!isStartWithSavePoint) {
+                        new SaveModeExecuteWrapper(handler).execute();
+                    } else {
+                        handler.handleSchemaSaveModeWithRestore();
+                    }
                 } catch (Exception e) {
                     throw new SeaTunnelRuntimeException(HANDLE_SAVE_MODE_FAILED, e);
                 }
             }
         } else if (sink instanceof MultiTableSink) {
             Map<TablePath, SeaTunnelSink> sinks = ((MultiTableSink) sink).getSinks();
             for (SeaTunnelSink seaTunnelSink : sinks.values()) {
-                handleSaveMode(seaTunnelSink);
+                handleSaveMode(seaTunnelSink, isStartWithSavePoint);
             }
         }
     }

Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,9 @@ public void handleDataSaveMode() {`
`55`	`55`	`log.info("handle data savemode with table path: {}", catalogTable.getTablePath());`
`56`	`56`	`}`
`57`	`57`
	`58`	`+ @Override`
	`59`	`+ public void handleSchemaSaveModeWithRestore() {}`
	`60`	`+`
`58`	`61`	`@Override`
`59`	`62`	`public SchemaSaveMode getSchemaSaveMode() {`
`60`	`63`	`return SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST;`
Original file line number	Diff line number	Diff line change
`@@ -271,7 +271,8 @@ public synchronized void init(long initializationTimestamp, boolean restart) thr`
`271`	`271`	`logicalVertexIdClassLoaderMap.get(`
`272`	`272`	`sink.getId()));`
`273`	`273`	`JobMaster.handleSaveMode(`
`274`		`- ((SinkAction<?, ?, ?, ?>) sink).getSink());`
	`274`	`+ ((SinkAction<?, ?, ?, ?>) sink).getSink(),`
	`275`	`+ logicalDag.isStartWithSavePoint());`
`275`	`276`	`});`
`276`	`277`	`Thread.currentThread().setContextClassLoader(appClassLoader);`
`277`	`278`	`}`
`@@ -556,22 +557,26 @@ public void run() {`
`556`	`557`	`}`
`557`	`558`	`}`
`558`	`559`
`559`		`- public static void handleSaveMode(SeaTunnelSink sink) {`
	`560`	`+ public static void handleSaveMode(SeaTunnelSink sink, boolean isStartWithSavePoint) {`
`560`	`561`	`if (sink instanceof SupportSaveMode) {`
`561`	`562`	`Optional<SaveModeHandler> saveModeHandler =`
`562`	`563`	`((SupportSaveMode) sink).getSaveModeHandler();`
`563`	`564`	`if (saveModeHandler.isPresent()) {`
`564`	`565`	`try (SaveModeHandler handler = saveModeHandler.get()) {`
`565`	`566`	`handler.open();`
`566`		`- new SaveModeExecuteWrapper(handler).execute();`
	`567`	`+ if (!isStartWithSavePoint) {`
	`568`	`+ new SaveModeExecuteWrapper(handler).execute();`
	`569`	`+ } else {`
	`570`	`+ handler.handleSchemaSaveModeWithRestore();`
	`571`	`+ }`
`567`	`572`	`} catch (Exception e) {`
`568`	`573`	`throw new SeaTunnelRuntimeException(HANDLE_SAVE_MODE_FAILED, e);`
`569`	`574`	`}`
`570`	`575`	`}`
`571`	`576`	`} else if (sink instanceof MultiTableSink) {`
`572`	`577`	`Map<TablePath, SeaTunnelSink> sinks = ((MultiTableSink) sink).getSinks();`
`573`	`578`	`for (SeaTunnelSink seaTunnelSink : sinks.values()) {`
`574`		`- handleSaveMode(seaTunnelSink);`
	`579`	`+ handleSaveMode(seaTunnelSink, isStartWithSavePoint);`
`575`	`580`	`}`
`576`	`581`	`}`
`577`	`582`	`}`