[WIP] Feature/kafka

.github/workflows/tests.yml

@@ -44,6 +44,27 @@ jobs:
         run: |
           go test ./test/... -config pebble.yml
 
+  kafkaTest:
+    needs: build
+    name: Kafka tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Python Dependencies for Conformance
+        run: pip install requests numpy kafka-python
+      - name: Download grip
+        uses: actions/download-artifact@v4
+        with:
+          name: gripBin
+      - name: Kafka Test
+        run: |
+          chmod +x grip
+          make start-kafka
+          ./grip server -c ./test/kafka.yml &
+          sleep 5
+          python conformance/run_kafka.py
+
 
   badgerTest:
     needs: build

Makefile

@@ -140,6 +140,52 @@ start-mysql:
 start-gripper-test:
 	@cd ./gripper/test-graph && ./gripper-table -m swapi/table.map &
 
+start-kafka:
+	@docker rm -f kafka > /dev/null 2>&1 || echo
+	docker run -d --name kafka \
+		-p 9092:9092 \
+		-e KAFKA_ENABLE_KRAFT=yes \
+		-e KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv== \
+		-e KAFKA_CFG_NODE_ID=1 \
+		-e KAFKA_CFG_PROCESS_ROLES=controller,broker \
+		-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \
+		-e KAFKA_CFG_LISTENERS=CONTROLLER://:9093,INTERNAL://:9092 \
+		-e KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://localhost:9092 \
+		-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT \
+		-e KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL \
+		-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
+		-e KAFKA_CFG_SUPER_USERS=User:admin \
+		-e KAFKA_CLIENT_USERS=admin \
+		-e KAFKA_CLIENT_PASSWORDS=adminpassword \
+		-e KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN \
+		-e KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN \
+		bitnami/kafka:latest
+	printf '%s\n' \
+		'security.protocol=SASL_PLAINTEXT' \
+		'sasl.mechanism=PLAIN' \
+		'sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="adminpassword";' \
+		> sasl-config.properties
+	docker cp sasl-config.properties kafka:/tmp/sasl-config.properties
+	@echo "Waiting for Kafka to become ready..."
+	@until docker exec kafka kafka-topics.sh \
+		--list \
+		--bootstrap-server localhost:9092 \
+		--command-config /tmp/sasl-config.properties > /dev/null 2>&1; do \
+		echo "Still waiting..."; \
+		sleep 2; \
+	done
+	docker exec kafka kafka-topics.sh \
+		--create \
+		--topic gripHistory \
+		--bootstrap-server localhost:9092 \
+		--partitions 1 \
+		--replication-factor 1 \
+		--command-config /tmp/sasl-config.properties
+	docker exec kafka kafka-topics.sh \
+		--list \
+		--bootstrap-server localhost:9092 \
+		--command-config /tmp/sasl-config.properties
+
 # ---------------------
 # Website
 # ---------------------
@@ -154,3 +200,4 @@ website-dev:
 # Other
 # ---------------------
 .PHONY: test rocksdb website
+

cmd/delete/main.go

@@ -3,7 +3,7 @@ package delete
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 
 	"github.com/bmeg/grip/gripql"
@@ -57,7 +57,7 @@ comma delimited --edges or --vertices arguments are also supported ex:
 			defer jsonFile.Close()
 
 			// Read the JSON file
-			byteValue, err := ioutil.ReadAll(jsonFile)
+			byteValue, err := io.ReadAll(jsonFile)
 			if err != nil {
 				log.Errorf("Failed to read file: %s", err)
 			}

cmd/mapping/main.go

@@ -2,7 +2,7 @@ package mapping
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 
 	"github.com/bmeg/grip/gripql"
@@ -75,7 +75,7 @@ var postCmd = &cobra.Command{
 			var graphs []*gripql.Graph
 			var err error
 			if jsonFile == "-" {
-				bytes, err := ioutil.ReadAll(os.Stdin)
+				bytes, err := io.ReadAll(os.Stdin)
 				if err != nil {
 					return err
 				}
@@ -98,7 +98,7 @@ var postCmd = &cobra.Command{
 			var graphs []*gripql.Graph
 			var err error
 			if jsonFile == "-" {
-				bytes, err := ioutil.ReadAll(os.Stdin)
+				bytes, err := io.ReadAll(os.Stdin)
 				if err != nil {
 					return err
 				}

cmd/schema/main.go

@@ -2,7 +2,7 @@ package schema
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 
 	"github.com/bmeg/grip/gripql"
@@ -82,7 +82,7 @@ var postCmd = &cobra.Command{
 			var graphs []*gripql.Graph
 			var err error
 			if jsonFile == "-" {
-				bytes, err := ioutil.ReadAll(os.Stdin)
+				bytes, err := io.ReadAll(os.Stdin)
 				if err != nil {
 					return err
 				}
@@ -98,15 +98,15 @@ var postCmd = &cobra.Command{
 				if err != nil {
 					return err
 				}
-				log.Debug("Posted schema: %s", g.Graph)
+				log.Debugf("Posted schema: %s", g.Graph)
 			}
 		}
 
 		if yamlFile != "" {
 			var graphs []*gripql.Graph
 			var err error
 			if jsonFile == "-" {
-				bytes, err := ioutil.ReadAll(os.Stdin)
+				bytes, err := io.ReadAll(os.Stdin)
 				if err != nil {
 					return err
 				}
@@ -135,7 +135,7 @@ var postCmd = &cobra.Command{
 				if err != nil {
 					return err
 				}
-				log.Debug("Posted schema: %s", g.Graph)
+				log.Debugf("Posted schema: %s", g.Graph)
 			}
 		}
 		if yamlSchemaPath != "" {
@@ -149,7 +149,7 @@ var postCmd = &cobra.Command{
 				if err != nil {
 					return err
 				}
-				log.Debug("Posted schema: %s", g.Graph)
+				log.Debugf("Posted schema: %s", g.Graph)
 			}
 		}
 		return nil