[wip] Master-slave deployment

.devcontainer/devcontainer.json

@@ -1,20 +1,18 @@
-
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node
 {
 	"name": "GraphScope",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
 	"image": "registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64",
-
 	// Features to add to the dev container. More info: https://containers.dev/features.
-    "features": {
-        "ghcr.io/devcontainers/features/common-utils:2":{
-            "installZsh": "true",
+	"features": {
+		"ghcr.io/devcontainers/features/common-utils:2": {
+			"installZsh": "true",
 			"configureZshAsDefaultShell": "true",
 			"installOhMyZsh": true,
-            "upgradePackages": "false"
-        }
-    },
+			"upgradePackages": "false"
+		}
+	},
 	// Configure tool-specific properties.
 	"customizations": {
 		// Configure properties specific to VS Code.
@@ -28,20 +26,15 @@
 			]
 		}
 	},
-
 	// Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "graphscope",
-
 	// Use 'postCreateCommand' to run commands before the container is created.
 	"initializeCommand": "sudo docker pull registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:latest",
-
 	// Uncomment this to enable C++ and Rust debugging in containers
 	// "capAdd": ["SYS_PTRACE"],
 	// "securityOpt": ["seccomp=unconfined"],
-
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [3000],
-
 	// Use 'portsAttributes' to set default properties for specific forwarded ports.
 	// More info: https://containers.dev/implementors/json_reference/#port-attributes
 	// "portsAttributes": {
@@ -50,23 +43,21 @@
 	// 		"onAutoForward": "notify"
 	// 	}
 	// },
-
 	// Use 'postCreateCommand' to run commands after the container is created.
 	// "postCreateCommand": "yarn install"
-
 	// Improve performance
-
 	// Uncomment these to mount a folder to a volume
 	// https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-targeted-named-volume
 	// "mounts": [
-    // 	"source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
+	// 	"source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
 	// ],
 	// "postCreateCommand": "sudo chown graphscope node_modules"
-
-
+	"runArgs": [
+		"--network=host",
+	],
 	// Uncomment these to use a named volume for your entire source tree
 	// https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-named-volume-for-your-entire-source-tree
 	// "workspaceMount": "source=gs,target=/workspaces,type=volume",
 	// "workspaceFolder": "/workspaces"
 	"postCreateCommand": "sudo chown -R graphscope /workspaces && bash pre-commit/install-hook.sh && bash pre-commit/prepare-commit-msg"
-}
+}

.github/workflows/flex-interactive.yml

@@ -70,7 +70,7 @@ jobs:
         docker image inspect graphscope/interactive:latest --format='{{.Size}}'
         # launch service: ${COORDINATOR_PORT} for coordinator http port; ${CYPHER_PORT} for cypher port;
         gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --cypher-port ${CYPHER_PORT} \
-              --coordinator-port ${COORDINATOR_PORT} --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml \
+              --coordinator-port ${COORDINATOR_PORT} --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml \
               --set storage.string_default_max_length=1024
         sleep 20
         # test
@@ -97,9 +97,9 @@ jobs:
       run: |
         # launch service: ${COORDINATOR_PORT} for coordinator http port; ${CYPHER_PORT} for cypher port;
         # replace max_content_length to 1MB
-        sed -i 's/max_content_length: .*/max_content_length: 1MB/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
+        sed -i 's/max_content_length: .*/max_content_length: 1MB/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml
         gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --cypher-port ${CYPHER_PORT} \
-              --coordinator-port ${COORDINATOR_PORT} --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml \
+              --coordinator-port ${COORDINATOR_PORT} --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml \
               --admin-port 7778 --storedproc-port 10001
         sleep 20
         # test
@@ -121,7 +121,7 @@ jobs:
         cd python && pip3 install -r requirements.txt && python3 setup.py build_proto
         python3 setup.py bdist_wheel
         pip3 install dist/*.whl
-        gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
+        gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml
         sleep 20
         # test
         cd ${GITHUB_WORKSPACE}/flex/interactive/sdk/examples/python

.github/workflows/interactive.yml

@@ -184,7 +184,23 @@ jobs:
         cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
         GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ${GS_TEST_DIR}
+        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml ${GS_TEST_DIR}
+
+        # test admin service with etcd metastore
+        GOOGLE_URL=https://storage.googleapis.com/etcd
+        GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
+        DOWNLOAD_URL=${GOOGLE_URL}
+        rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test
+        curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+        tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test
+        /tmp/etcd-download-test/etcd &
+        sleep 3
+
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
+        python3 -c 'import yaml;f=open("./interactive_config_standalone.yaml");y=yaml.safe_load(f);y["compute_engine"]["metadata_store"]["uri"] = "http://localhost:2379"; f.close();f=open("./interactive_config_test_etcd.yaml","w");yaml.dump(y,f);f.close()'
+        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test_etcd.yaml ${GS_TEST_DIR}
+        rm ./interactive_config_test_etcd.yaml
+
 
     - name: Test Interactive Python Admin Service
       run:
@@ -216,19 +232,19 @@ jobs:
         cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
         GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        sed -i 's/interactive_workspace/temp_workspace/g' ./interactive_config_test.yaml
-        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml java
+        sed -i 's/interactive_workspace/temp_workspace/g' ./interactive_config_standalone.yaml
+        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml java
         export ENGINE_TYPE=interactive
-        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml python
-        sed -i 's/temp_workspace/interactive_workspace/g' ./interactive_config_test.yaml
+        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml python
+        sed -i 's/temp_workspace/interactive_workspace/g' ./interactive_config_standalone.yaml
 
     - name: Robustness test 
       env:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
         GS_TEST_DIR: ${{ github.workspace }}/gstest
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml
+        bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml ./interactive_config_standalone_cbo.yaml
 
     - name: Sample Query test
       env:
@@ -251,15 +267,15 @@ jobs:
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
         pip3 install argparse
         pip3 install neo4j
-        bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml
+        bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml
 
     - name: Run codegen test.
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        bash hqps_codegen_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml
+        bash hqps_codegen_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_standalone.yaml ./interactive_config_standalone_cbo.yaml
 
     - name: Test cypher&cpp procedure generation and loading
       env:
@@ -280,18 +296,18 @@ jobs:
 
         cd ${GITHUB_WORKSPACE}/flex/bin
         ./load_plan_and_gen.sh -e=hqps -i=../tests/interactive/plus_one.cc -w=/tmp/codegen \
-          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \
+          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml -o=${PLUGIN_DIR} \
           --procedure_name=plus_one \
           --graph_schema_path=../interactive/examples/modern_graph/graph.yaml \
           --procedure_desc="This is test procedure, and the input is a number, and the output is the number plus one."
 
         ./load_plan_and_gen.sh -e=hqps -i=../interactive/sdk/java/src/test/resources/sample_app.cc -w=/tmp/codegen \
-          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \
+          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml -o=${PLUGIN_DIR} \
           --procedure_name=sample_app \
           --graph_schema_path=../interactive/examples/modern_graph/graph.yaml
 
         ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
-          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml  -o=${PLUGIN_DIR} \
+          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml  -o=${PLUGIN_DIR} \
           --procedure_name=count_vertex_num \
           --graph_schema_path=../interactive/examples/modern_graph/graph.yaml
 
@@ -300,12 +316,12 @@ jobs:
         # plus_one: (num: int64) -> (num: int64), CppEncoder
         # sample_app: (num: int64) -> (num: int64), kCypherJson
 
-        sed -i 's/interactive_workspace/temp_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
+        sed -i 's/interactive_workspace/temp_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml
         cd ${GITHUB_WORKSPACE}/flex/tests/interactive/
         bash test_plugin_loading.sh ${TMP_INTERACTIVE_WORKSPACE} modern_graph \
-           ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml \
+           ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml \
            ./modern_graph_schema_v0_0.yaml  ./modern_graph_schema_v0_1.yaml
-        sed -i 's/temp_workspace/interactive_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
+        sed -i 's/temp_workspace/interactive_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_standalone.yaml
 
     - name: Let compiler use latest interactive java sdk
       env:
@@ -476,7 +492,6 @@ jobs:
         git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
         cd /tmp/libgrape-lite
         mkdir -p build && cd build
-        cmake ..
         make -j$(nproc)
         make install
 
@@ -641,4 +656,4 @@ jobs:
         SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
         BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
         sed -i 's/|/\\t/g' ${BULK_LOAD_FILE}
-        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
+        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

.gitmodules

@@ -13,6 +13,15 @@
 [submodule "flex/third_party/parallel-hashmap"]
 	path = flex/third_party/parallel-hashmap
 	url = https://github.com/greg7mdp/parallel-hashmap.git
+
+[submodule "flex/third_party/etcd-cpp-apiv3"]
+	path = flex/third_party/etcd-cpp-apiv3
+	url = https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git
+
+[submodule "flex/third_party/cpprestsdk"]
+	path = flex/third_party/cpprestsdk
+	url = https://github.com/microsoft/cpprestsdk.git
+
 [submodule "flex/third_party/aliyun-oss-cpp-sdk"]
 	path = flex/third_party/aliyun-oss-cpp-sdk
 	url = https://github.com/aliyun/aliyun-oss-cpp-sdk.git

charts/graphscope-interactive/templates/_helpers.tpl

@@ -156,7 +156,7 @@ Return the configmap with the graphscope configuration
 Return the engineConfigPath with the graphscope configuration
 */}}
 {{- define "graphscope-interactive.engineConfigPath" -}}
-/etc/interactive/interactive_config.yaml
+/opt/flex/share/interactive_config.yaml
 {{- end -}}
 
 {{/*

charts/graphscope-interactive/values.yaml

@@ -18,6 +18,8 @@ commonAnnotations: {}
 ##
 commonLabels: {}
 
+imagePullSecrets: {}
+
 ##
 javaOpts: ""
 
@@ -30,12 +32,10 @@ workspace: "/tmp/interactive_workspace"
 ## default graph
 defaultGraph: modern_graph
 
-
 hiactorWorkerNum: 1
 
 hiactorTimeout: 240000
 
-
 # ## need by vineyard in distributed env
 # etcdEndpoint: "etcd-for-vineyard.default.svc.cluster.local:2379"
 
@@ -73,7 +73,7 @@ persistence:
   ##
   storageClass: ""
   accessModes:
-  - ReadWriteOnce
+    - ReadWriteOnce
   annotations: {}
   ## @param persistence.labels Labels for the PVC
   ##
@@ -98,7 +98,7 @@ engine:
     ## pullSecrets:
     ##   - myRegistryKeySecretName
     ##
-    pullSecrets: [ ]
+    pullSecrets: []
 
   replicaCount: 1
 
@@ -133,6 +133,7 @@ engine:
   ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
   ## Note: podAffinityPreset, podAntiAffinityPreset, and  nodeAffinityPreset will be ignored when it's set
   ##
+  affinity: {}
   # affinity:
   #   nodeAffinity:
   #     requiredDuringSchedulingIgnoredDuringExecution:
@@ -170,7 +171,8 @@ engine:
   ## GraphScope Interactive container's resource requests and limits
   ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
   ##
-  resources: {}
+  resources:
+    {}
     # We usually recommend not to specify default resources and to leave this as a conscious
     # choice for the user. This also increases chances charts run on environments with little
     # resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -319,7 +321,7 @@ frontend:
     ## pullSecrets:
     ##   - myRegistryKeySecretName
     ##
-    pullSecrets: [ ]
+    pullSecrets: []
 
   replicaCount: 1
 
@@ -373,7 +375,8 @@ frontend:
   ## GraphScope Interactive container's resource requests and limits
   ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
   ##
-  resources: {}
+  resources:
+    {}
     # We usually recommend not to specify default resources and to leave this as a conscious
     # choice for the user. This also increases chances charts run on environments with little
     # resources, such as Minikube. If you do want to specify resources, uncomment the following

charts/interactive/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/interactive/Chart.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+name: interactive
+description: GraphScope Interactive
+icon: https://graphscope.io/assets/images/graphscope-logo.svg
+home: https://graphscope.io
+
+sources:
+  - https://github.com/alibaba/GraphScope/flex/interactive/
+
+keywords:
+  - GraphScope
+  - Interactive
+  - Graph Analytics
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.2
+
+dependencies:
+  - name: kafka
+    repository: oci://registry-1.docker.io/bitnamicharts
+    version: "20.0.6"
+  - name: etcd
+    repository: oci://registry-1.docker.io/bitnamicharts
+    version: "11.1.3"