feat: logs scraper

.editorconfig

@@ -1,3 +1,16 @@
+root = true
+
 [*.ts]
 indent_style = space
 indent_size = 2
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{yaml,yml}]
+indent_style = space
+indent_size = 2
+quote_type = single

.github/workflows/test.yml

@@ -28,6 +28,61 @@ jobs:
       - name: Test
         run: make test
 
+  e2e:
+    runs-on: ubuntu-latest
+    services:
+      loki:
+        image: grafana/loki:3.5.1
+        ports:
+          - 3100:3100
+        options: >-
+          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      opensearch:
+        image: opensearchproject/opensearch:2.11.1
+        ports:
+          - 9200:9200
+          - 9600:9600
+        env:
+          discovery.type: single-node
+          plugins.security.disabled: true
+          bootstrap.memory_lock: true
+          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m -XX:MaxMetaspaceSize=128m
+        options: >-
+          --health-cmd "curl -f http://localhost:9200/_cluster/health || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 30s
+          --ulimit memlock=-1:-1
+          --ulimit nofile=65536:65536
+    steps:
+      - name: Install Go
+        uses: buildjet/setup-go@v5
+        with:
+          go-version: 1.24.x
+      - name: Checkout code
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
+      - uses: buildjet/cache@v4
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            .bin
+          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
+          restore-keys: |
+            cache-
+      - name: E2E Test
+        run: |
+          make ginkgo
+          ginkgo -r tests/e2e/
+        env:
+          DUTY_DB_DISABLE_RLS: 'true'
+          LOKI_URL: http://localhost:3100
+          OPENSEARCH_URL: http://localhost:9200
+
   test-prod:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -55,57 +110,56 @@ jobs:
   test-clickhouse:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-    - name: Kubernetes KinD Cluster
-      uses: container-tools/kind-action@v2
+      - name: Kubernetes KinD Cluster
+        uses: container-tools/kind-action@v2
 
-    - name: Install Helm
-      uses: azure/setup-helm@v3
+      - name: Install Helm
+        uses: azure/setup-helm@v3
 
-    - name: Verify cluster is ready
-      run: |
-        echo "Waiting for cluster to be ready..."
-        kubectl wait --for=condition=Ready nodes --all --timeout=300s
-        kubectl get nodes
-        kubectl get pods -A
+      - name: Verify cluster is ready
+        run: |
+          echo "Waiting for cluster to be ready..."
+          kubectl wait --for=condition=Ready nodes --all --timeout=300s
+          kubectl get nodes
+          kubectl get pods -A
 
-    - name: Build and push Docker image
-      uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
-      with:
-        context: .
-        file: ./build/Dockerfile
-        push: true
-        tags: localhost:5000/flanksource/config-db:latest
-        cache-from: type=registry,ref=docker.io/flanksource/config-db
+      - name: Build and push Docker image
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./build/Dockerfile
+          push: true
+          tags: localhost:5000/flanksource/config-db:latest
+          cache-from: type=registry,ref=docker.io/flanksource/config-db
 
-    - name: Package helm chart
-      run: |
-        helm dependency build ./chart
-        helm package ./chart --version 1.0.0
+      - name: Package helm chart
+        run: |
+          helm dependency build ./chart
+          helm package ./chart --version 1.0.0
 
-    - name: Install Helm chart
-      run: |
-        helm install config-db config-db-1.0.0.tgz \
-          --namespace default \
-          --set clickhouse.enabled=true \
-          --set imageRegistry="kind-registry:5000"
+      - name: Install Helm chart
+        run: |
+          helm install config-db config-db-1.0.0.tgz \
+            --namespace default \
+            --set clickhouse.enabled=true \
+            --set imageRegistry="kind-registry:5000"
 
-    - name: Check pod status
-      run: |
+      - name: Check pod status
+        run: |
           sleep 60
           kubectl get pods
           kubectl describe pods
           sleep 60
           kubectl get pods
           kubectl describe pods
-
 
-    - name: Run clickhouse fixture
-      run: |
-        kubectl cp fixtures/clickhouse.yaml config-db-0:/app/clickhouse.yaml
-        kubectl exec -it config-db-0 -- /app/config-db run /app/clickhouse.yaml
+      - name: Run clickhouse fixture
+        run: |
+          kubectl cp fixtures/clickhouse.yaml config-db-0:/app/clickhouse.yaml
+          kubectl exec -it config-db-0 -- /app/config-db run /app/clickhouse.yaml
 
   test-load:
     if: false # disabled for now

Makefile

@@ -32,10 +32,12 @@ gen-schemas:
 	go mod edit -module=github.com/flanksource/config-db/hack/generate-schemas && \
 	go mod edit -require=github.com/flanksource/[email protected] && \
 	go mod edit -replace=github.com/flanksource/config-db=../../ && \
+	if grep -v "^//" ../../go.mod | grep -q "replace.*github.com/flanksource/duty.*=>"; then \
+		go mod edit -replace=github.com/flanksource/duty=../../../duty; \
+	fi && \
 	go mod tidy && \
 	go run ./main.go
 
-
 docker:
 	docker build . -f build/Dockerfile -t ${IMG}
 
@@ -70,22 +72,41 @@ test-load:
 	$(MAKE) gotest-load
 
 .PHONY: gotest
-gotest:
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
+gotest: ginkgo
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		ginkgo -r -v --skip-package=tests/e2e -coverprofile cover.out ./...
 
 .PHONY: gotest-prod
 gotest-prod:
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -tags rustdiffgen ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		go test -tags rustdiffgen ./... -coverprofile cover.out -skip "TestE2E"
 
 .PHONY: gotest-load
 gotest-load:
 	make -C fixtures/load k6
-	LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./tests -coverprofile cover.out
+	LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		go test -v ./tests -coverprofile cover.out -skip "TestE2E"
 
 .PHONY: env
 env: envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		ginkgo -r -v --skip-package=tests/e2e -coverprofile cover.out
+
+.PHONY: ginkgo
+ginkgo:
+	go install github.com/onsi/ginkgo/v2/ginkgo
+
+.PHONY: test-e2e
+test-e2e: ginkgo
+	cd tests/e2e && docker-compose up -d && \
+	echo 'Running tests' && \
+	(ginkgo -v; TEST_EXIT_CODE=$$?; docker-compose down; exit $$TEST_EXIT_CODE)
 
+.PHONY: e2e-services
+e2e-services: ## Run e2e test services in foreground with automatic cleanup on exit
+	cd tests/e2e && \
+	trap 'docker-compose down -v && docker-compose rm -f' EXIT INT TERM && \
+	docker-compose up --remove-orphans
 
 fmt:
 	go fmt ./...
@@ -155,10 +176,6 @@ dev:
 watch:
 	watchexec -c make build install
 
-.PHONY: test-e2e
-test-e2e: bin
-	./test/e2e.sh
-
 .bin/upx: .bin
 	wget -nv -O upx.tar.xz https://github.com/upx/upx/releases/download/v3.96/upx-3.96-$(ARCH)_$(OS).tar.xz
 	tar xf upx.tar.xz

api/v1/logs.go

@@ -0,0 +1,43 @@
+package v1
+
+import (
+	"github.com/flanksource/duty/connection"
+	"github.com/flanksource/duty/logs"
+	"github.com/flanksource/duty/logs/gcpcloudlogging"
+	"github.com/flanksource/duty/logs/loki"
+	"github.com/flanksource/duty/logs/opensearch"
+)
+
+type Logs struct {
+	BaseScraper `json:",inline"`
+
+	// Loki specifies the Loki configuration for log scraping
+	Loki *LokiConfig `json:"loki,omitempty"`
+
+	// GCPCloudLogging specifies the GCP Cloud Logging configuration
+	GCPCloudLogging *GCPCloudLoggingConfig `json:"gcpCloudLogging,omitempty"`
+
+	// OpenSearch specifies the OpenSearch configuration for log scraping
+	OpenSearch *OpenSearchConfig `json:"openSearch,omitempty"`
+
+	// FieldMapping defines how source log fields map to canonical LogLine fields
+	FieldMapping *logs.FieldMappingConfig `json:"fieldMapping,omitempty"`
+}
+
+// LokiConfig contains configuration for Loki log scraping
+type LokiConfig struct {
+	connection.Loki `json:",inline"`
+	loki.Request    `json:",inline"`
+}
+
+// GCPCloudLoggingConfig contains configuration for GCP Cloud Logging
+type GCPCloudLoggingConfig struct {
+	connection.GCPConnection `json:",inline"`
+	gcpcloudlogging.Request  `json:",inline"`
+}
+
+// OpenSearchConfig contains configuration for OpenSearch log scraping
+type OpenSearchConfig struct {
+	opensearch.Backend `json:",inline"`
+	opensearch.Request `json:",inline"`
+}

api/v1/types.go

@@ -22,6 +22,7 @@ var AllScraperConfigs = map[string]any{
 	"http":           HTTP{},
 	"kubernetes":     Kubernetes{},
 	"kubernetesfile": KubernetesFile{},
+	"logs":           Logs{},
 	"slack":          Slack{},
 	"sql":            SQL{},
 	"terraform":      Terraform{},
@@ -65,6 +66,7 @@ type ScraperSpec struct {
 	Terraform      []Terraform      `json:"terraform,omitempty" yaml:"trivy,omitempty"`
 	HTTP           []HTTP           `json:"http,omitempty"`
 	Clickhouse     []Clickhouse     `json:"clickhouse,omitempty"`
+	Logs           []Logs           `json:"logs,omitempty"`
 
 	// CRDSync when set to true, will create (or update) the corresponding database record
 	// for a config item of the following types
@@ -132,6 +134,10 @@ func (c ScraperSpec) ApplyPlugin(plugins []ScrapePluginSpec) ScraperSpec {
 		spec.HTTP[i].BaseScraper = spec.HTTP[i].BaseScraper.ApplyPlugins(plugins...)
 	}
 
+	for i := range spec.Logs {
+		spec.Logs[i].BaseScraper = spec.Logs[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
 	return *spec
 }