Merge EventMesh function branch to master (#4941)

* EventMesh function admin (#4851)

* own

* dependency

* finish registry

* EventMesh function admin (#4853)

* own

* dependency

* finish registry

* init

* Eventmesh function admin (#4854)

* own

* dependency

* finish registry

* init

* 0419

* 0419

* more discovery and move gRPC

* fix dependency

* EventMesh function connector runtime (#4858)

* [ISSUE #4812] Set up Admin Endpoints v2 (#4813)

* Remove redundant overloaded methods

* Simplify write() result param

* Add writeJson(); Add PUT; Add JavaDoc

* Rename EventHttpHandler to EventMeshHttpHandler

* Correct server thread name

* Clean up messy & non-hierarchical overloading

* No need to set headers manually any more

* Set up v1&v2 endpoints

* Set up v1&v2 response dto

* Introduce fastjson2

* Fix fastjson2 "level too large : 2048" error caused by IPAddress

* Correct @ConfigField naming

* Return properties format json key

* Add format option to query string

* Introduce Result

* Reduce duplicate builder code

* Fix all checkstyle warnings in eventmesh-runtime

* Add known dependency

* [ISSUE #4814] Migrate from fastjson 1.2.83 to fastjson2 (#4819)

* [Enhancement] Migrate from fastjson 1.2.83 to fastjson2 #4814

* fix_dependencies_problem

* fix_check

* [ISSUE #4551] modify the logic of time-consumption statistics (#4822)

* init connector runtime v2

* [ISSUE #4804] Fix SubStreamHandler exception loop by closeOnError (#4807)

* Handle exception loop by closeOnError

* Lombok optimization

* some format optimization

* Avoid closing multiple times

* Remove redundant set null

* Revert "Avoid closing multiple times"

This reverts commit 774397f.

* Use synchronized latch to keep senderOnComplete called once

* Use boolean to prevent latch called by somebody else

* Remove the unique callee/caller close() of onCompleted()

* [ISSUE #4838] Deprecate unused `eventMesh.connector.plugin.type` etc. properties (#4839)

* Remove all references of `eventMesh.connector.plugin.type`

* Deprecate `eventMesh.connector.plugin.type` and sort properties

* Remove misconfigured & not-used `registerIntervalInMills`, `fetchRegistryAddrIntervalInMills`

* Remove 'defibus' related un-used usages

* Supplement #4809 for `null != object`

* [ISSUE #4832] Downgrade stale bot to v8 to resolve state cache reserving error (#4833)

* Revert stale bot to v8 to resolve state cache reserving error

* Reduce operations-per-run to default value to ease pressure

* Unify yaml to yml

* [ISSUE #4820] Bug fix EventHandler not return json (#4821)

* bug fix

* bug fix

* bug fix

* update runtime v2

* update connector runtime

* update connector runtime

* update connector runtime

* update connector runtime

* update connector runtime

---------

Co-authored-by: Pil0tXia <[email protected]>
Co-authored-by: Zaki <[email protected]>
Co-authored-by: Karson <[email protected]>

* [ISSUE #4931]Add Registry Module for Discovery AdminServer

* [ISSUES #4933]Add Admin Module

* [ISSUE #4935] Add and Move the Pojo Used By Both Runtime and Admin to Common

* [ISSUE #4937]fix gradle dependecy and add runtime v2

* [ISSUES #4939]add canal connector

* fix missing apache header

* fix missing apache header

* fix missing apache header

* update gradle dependencies

* fix admin server ci check error

* fix admin server ci check error

* fix ci checkStyle error

* fix ci check error

---------

Co-authored-by: sodaRyCN <[email protected]>
Co-authored-by: Pil0tXia <[email protected]>
Co-authored-by: Zaki <[email protected]>
Co-authored-by: Karson <[email protected]>

Author: 5 people

Diff for: .github/workflows/docker.yml

@@ -47,4 +47,4 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           file: ./docker/Dockerfile_jdk8
-          context: ./
+          context: ./

Diff for: .idea/icon.png

@@ -95,7 +95,8 @@ allprojects {
 
     checkstyleMain.exclude('**/org/apache/eventmesh/client/grpc/protos**')
             .exclude('**/org/apache/eventmesh/common/protocol/grpc/cloudevents**')
-            .exclude('**/org/apache/eventmesh/common/protocol/grpc/protos/**')
+            .exclude('**/org/apache/eventmesh/common/protocol/grpc/proto**')
+            .exclude('**/org/apache/eventmesh/common/protocol/grpc/adminserver/**')
             .exclude('**/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService**')
             .exclude('**/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc**')
             .exclude('**/org/apache/eventmesh/connector/jdbc/antlr**')
@@ -405,7 +406,7 @@ tasks.register('checkDeniedLicense') {
                 "BSD-4-Clause", "BSD-4-Clause-UC", "NPL-1.0", "NPL-1.1", "JSON"
         ]
         // Update exemptions according to https://github.com/apache/eventmesh/issues/4842
-        def allowedArtifacts = ["amqp-client", "stax-api", "javassist", "ST4", "xsdlib"]
+        def allowedArtifacts = ["amqp-client", "stax-api", "javassist", "ST4", "xsdlib", "jsqlparser"]
 
         def licenseFile = file('tools/dist-license/LICENSE')
         def lines = licenseFile.readLines()
@@ -751,6 +752,13 @@ subprojects {
 
             dependency "software.amazon.awssdk:s3:2.25.55"
             dependency "com.github.rholder:guava-retrying:2.0.0"
+
+            dependency "org.mybatis.spring.boot:mybatis-spring-boot-starter:2.3.1"
+            dependency "com.alibaba:druid-spring-boot-starter:1.2.22"
+            dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.10"
+
+            dependency "com.baomidou:mybatis-plus:3.5.6"
+            dependency "com.mysql:mysql-connector-j:8.0.33"
         }
     }
-}
+}

Diff for: build.gradle

@@ -0,0 +1,42 @@
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store

Diff for: eventmesh-admin-server/.gitignore

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+dependencies {
+	implementation project(":eventmesh-spi")
+	implementation project(":eventmesh-common")
+	implementation project(":eventmesh-registry:eventmesh-registry-api")
+	implementation project(":eventmesh-registry:eventmesh-registry-nacos")
+	implementation project(':eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api')
+	implementation "com.alibaba.nacos:nacos-client"
+	implementation ("org.springframework.boot:spring-boot-starter-web") {
+		exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
+	}
+	implementation 'org.springframework.boot:spring-boot-starter-jetty'
+	implementation "io.grpc:grpc-core"
+	implementation "io.grpc:grpc-protobuf"
+	implementation "io.grpc:grpc-stub"
+	implementation "io.grpc:grpc-netty-shaded"
+
+	// https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter
+	implementation group: 'com.baomidou', name: 'mybatis-plus-boot-starter', version: '3.5.5'
+	implementation "org.reflections:reflections:0.10.2"
+
+	// https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter
+	implementation "com.alibaba:druid-spring-boot-starter"
+	compileOnly 'com.mysql:mysql-connector-j'
+	compileOnly 'org.projectlombok:lombok'
+	annotationProcessor 'org.projectlombok:lombok'
+}
+
+configurations.implementation  {
+	exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
+}
+

Diff for: eventmesh-admin-server/build.gradle

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

Diff for: eventmesh-admin-server/gradle.properties

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.admin.server;
+
+import org.apache.eventmesh.common.remote.Task;
+import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest;
+import org.apache.eventmesh.common.utils.PagedList;
+
+/**
+ * Admin
+ */
+public interface Admin extends ComponentLifeCycle {
+
+    /**
+     * support for web or ops
+     **/
+    boolean createOrUpdateTask(Task task);
+
+    boolean deleteTask(Long id);
+
+    Task getTask(Long id);
+
+    // paged list
+    PagedList<Task> getTaskPaged(Task task);
+
+    /**
+     * support for task
+     */
+    void reportHeartbeat(ReportHeartBeatRequest heartBeat);
+}

Diff for: eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/Admin.java

@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.admin.server;
+
+import org.apache.eventmesh.common.Constants;
+import org.apache.eventmesh.common.config.CommonConfiguration;
+import org.apache.eventmesh.common.config.ConfigService;
+import org.apache.eventmesh.common.remote.Task;
+import org.apache.eventmesh.common.remote.exception.ErrorCode;
+import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest;
+import org.apache.eventmesh.common.utils.IPUtils;
+import org.apache.eventmesh.common.utils.PagedList;
+import org.apache.eventmesh.registry.RegisterServerInfo;
+import org.apache.eventmesh.registry.RegistryFactory;
+import org.apache.eventmesh.registry.RegistryService;
+
+import org.apache.commons.lang3.StringUtils;
+
+import javax.annotation.PostConstruct;
+
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.ApplicationListener;
+import org.springframework.stereotype.Service;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+public class AdminServer implements Admin, ApplicationListener<ApplicationReadyEvent> {
+
+    private final RegistryService registryService;
+
+    private final RegisterServerInfo adminServeInfo;
+
+    private final CommonConfiguration configuration;
+
+    public AdminServer(AdminServerProperties properties) {
+        configuration =
+            ConfigService.getInstance().buildConfigInstance(CommonConfiguration.class);
+        if (configuration == null) {
+            throw new AdminServerRuntimeException(ErrorCode.STARTUP_CONFIG_MISS, "common configuration file miss");
+        }
+        this.adminServeInfo = new RegisterServerInfo();
+
+        adminServeInfo.setHealth(true);
+        adminServeInfo.setAddress(IPUtils.getLocalAddress() + ":" + properties.getPort());
+        String name = Constants.ADMIN_SERVER_REGISTRY_NAME;
+        if (StringUtils.isNotBlank(properties.getServiceName())) {
+            name = properties.getServiceName();
+        }
+        adminServeInfo.setServiceName(name);
+        registryService = RegistryFactory.getInstance(configuration.getEventMeshRegistryPluginType());
+    }
+
+
+    @Override
+    public boolean createOrUpdateTask(Task task) {
+        return false;
+    }
+
+    @Override
+    public boolean deleteTask(Long id) {
+        return false;
+    }
+
+    @Override
+    public Task getTask(Long id) {
+        return null;
+    }
+
+    @Override
+    public PagedList<Task> getTaskPaged(Task task) {
+        return null;
+    }
+
+    @Override
+    public void reportHeartbeat(ReportHeartBeatRequest heartBeat) {
+
+    }
+
+    @Override
+    @PostConstruct
+    public void start() {
+        if (configuration.isEventMeshRegistryPluginEnabled()) {
+            registryService.init();
+        }
+    }
+
+    @Override
+    public void destroy() {
+        if (configuration.isEventMeshRegistryPluginEnabled()) {
+            registryService.unRegister(adminServeInfo);
+            try {
+                Thread.sleep(3000);
+            } catch (InterruptedException ignore) {
+                log.warn("interrupted when sleep");
+                Thread.currentThread().interrupt();
+            }
+            registryService.shutdown();
+        }
+    }
+
+    @Override
+    public void onApplicationEvent(ApplicationReadyEvent event) {
+        if (configuration.isEventMeshRegistryPluginEnabled()) {
+            log.info("application is started and registry plugin is enabled, it's will register admin self");
+            registryService.register(adminServeInfo);
+        }
+    }
+}

Diff for: eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.admin.server;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@ConfigurationProperties("event-mesh.admin-server")
+@Getter
+@Setter
+public class AdminServerProperties {
+
+    private int port;
+    private boolean enableSSL;
+    private String configurationPath;
+    private String configurationFile;
+    private String serviceName;
+}

Diff for: eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.admin.server;
+
+import lombok.Getter;
+
+public class AdminServerRuntimeException extends RuntimeException {
+
+    @Getter
+    private final int code;
+
+    public AdminServerRuntimeException(int code, String message) {
+        super(message);
+        this.code = code;
+    }
+}