Enable running ITs against Quesma with debugger (#1479)

<!-- A note on testing your PR -->
<!-- Basic unit test run is executed against each commit in the PR.
If you want to run a full integration test suite, you can trigger it by
commenting
     with '/run-integration-tests' or '/run-it' -->

Author: mieciu

.gitignore

@@ -1,6 +1,13 @@
 ci/it/configs/*.yml
 elasticsearch-data
-.idea
+.idea/*
+!.idea/runConfigurations/
+!.idea/runConfigurations/*.xml
+# User-specific IDEA files
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/usage.statistics.xml
+.idea/dictionaries/
 __pycache__
 mitmproxy/requests/*.txt
 mitmproxy/requests/*.mitm

.idea/runConfigurations/Debug_Quesma_ITs.xml

@@ -0,0 +1,19 @@
+Quesma Integration Tests
+========================
+
+This directory contains integration tests for Quesma. 
+These are simple, end-to-end tests that verify the functionality of Quesma using [Testcontainers library](https://testcontainers.com).
+
+
+
+
+How to debug
+============
+
+There is a way to run these tests agains a local Quesma instance with debugger attached.
+
+1. Set up a breakpoint in Quesma codebase.
+2. Change the `debugQuesmaDuringTestRun` flag to `true` in `ci/it/testcases/utils.go`
+3. Start an integration test (either with CLI or in your IDE using play button next to the test declaration).
+   ...Test case execution will block and wait until you start Quesma manually in IDE in debug mode)
+4. Start Quesma in Debug mode using `Debug Quesma ITs` Run Configuration in your IDE.

ci/it/README.md

@@ -43,6 +43,9 @@ func (tc *IntegrationTestcaseBase) Cleanup(ctx context.Context, t *testing.T) {
 }
 
 func (tc *IntegrationTestcaseBase) getQuesmaEndpoint() string {
+	if debugQuesmaDuringTestRun { // If debug mode is enabled, return a hardcoded endpoint for Quesma
+		return "http://localhost:8080"
+	}
 	ctx := context.Background()
 	q := *tc.Containers.Quesma
 	p, _ := q.MappedPort(ctx, "8080/tcp")

ci/it/testcases/base.go

@@ -0,0 +1,132 @@
+// Copyright Quesma, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+
+package testcases
+
+import (
+	"context"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-connections/nat"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/exec"
+	"io"
+	"time"
+)
+
+func NewManuallyCreatedContainer() *ManuallyCreatedContainer {
+	return &ManuallyCreatedContainer{}
+}
+
+type ManuallyCreatedContainer struct{}
+
+func (c ManuallyCreatedContainer) GetContainerID() string {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Endpoint(ctx context.Context, s string) (string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) PortEndpoint(ctx context.Context, port nat.Port, s string) (string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Host(ctx context.Context) (string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Inspect(ctx context.Context) (*types.ContainerJSON, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) MappedPort(ctx context.Context, port nat.Port) (nat.Port, error) {
+	return "8080/tcp", nil
+}
+
+func (c ManuallyCreatedContainer) Ports(ctx context.Context) (nat.PortMap, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) SessionID() string {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) IsRunning() bool {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Start(ctx context.Context) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Stop(ctx context.Context, duration *time.Duration) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Terminate(ctx context.Context) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Logs(ctx context.Context) (io.ReadCloser, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) FollowOutput(consumer testcontainers.LogConsumer) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) StartLogProducer(ctx context.Context, option ...testcontainers.LogProductionOption) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) StopLogProducer() error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Name(ctx context.Context) (string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) State(ctx context.Context) (*types.ContainerState, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Networks(ctx context.Context) ([]string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) NetworkAliases(ctx context.Context) (map[string][]string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) Exec(ctx context.Context, cmd []string, options ...exec.ProcessOption) (int, io.Reader, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) ContainerIP(ctx context.Context) (string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) ContainerIPs(ctx context.Context) ([]string, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) CopyFileFromContainer(ctx context.Context, filePath string) (io.ReadCloser, error) {
+	panic("implement me")
+}
+
+func (c ManuallyCreatedContainer) GetLogProductionErrorChannel() <-chan error {
+	panic("implement me")
+}

ci/it/testcases/container_utils.go

@@ -13,6 +13,7 @@ import (
 	"github.com/testcontainers/testcontainers-go/wait"
 	"io"
 	"log"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -21,12 +22,21 @@ import (
 	"time"
 )
 
+const (
+	// debugQuesmaDuringTestRun should be set to `true` if you would like to run Quesma in IDE with debugger on
+	// during the integration test run.
+	debugQuesmaDuringTestRun = false
+)
+
 const configTemplatesDir = "configs"
 
 func GetInternalDockerHost() string {
 	if check := os.Getenv("EXECUTING_ON_GITHUB_CI"); check != "" {
 		return "localhost-for-github-ci"
 	}
+	if debugQuesmaDuringTestRun {
+		return "localhost"
+	}
 	return "host.docker.internal" // `host.testcontainers.internal` doesn't work for Docker Desktop for Mac.
 }
 
@@ -264,26 +274,27 @@ func setupClickHouse(ctx context.Context) (testcontainers.Container, error) {
 	})
 }
 
-func RenderQuesmaConfig(configTemplate string, data map[string]string) error {
+func RenderQuesmaConfig(configTemplate string, data map[string]string) (string, error) {
 	absPath, err := filepath.Abs(filepath.Join(".", configTemplatesDir, configTemplate))
 	content, err := os.ReadFile(absPath)
 	if err != nil {
-		return fmt.Errorf("error reading YAML file: %v", err)
+		return "", fmt.Errorf("error reading YAML file: %v", err)
 	}
 	tmpl, err := template.New("yamlTemplate").Parse(string(content))
 	if err != nil {
-		return fmt.Errorf("error creating template: %v", err)
+		return "", fmt.Errorf("error creating template: %v", err)
 	}
 	var renderedContent bytes.Buffer
 	err = tmpl.Execute(&renderedContent, data)
 	if err != nil {
-		return fmt.Errorf("error executing template: %v", err)
+		return "", fmt.Errorf("error executing template: %v", err)
 	}
-	err = os.WriteFile(strings.TrimSuffix(absPath, ".template"), renderedContent.Bytes(), 0644)
+	configPath := strings.TrimSuffix(absPath, ".template")
+	err = os.WriteFile(configPath, renderedContent.Bytes(), 0644)
 	if err != nil {
-		return fmt.Errorf("error writing rendered YAML file: %v", err)
+		return "", fmt.Errorf("error writing rendered YAML file: %v", err)
 	}
-	return nil
+	return configPath, nil
 }
 
 func setupContainersForTransparentProxy(ctx context.Context, quesmaConfigTemplate string) (*Containers, error) {
@@ -300,7 +311,7 @@ func setupContainersForTransparentProxy(ctx context.Context, quesmaConfigTemplat
 		"elasticsearch_host": GetInternalDockerHost(),
 		"elasticsearch_port": esPort.Port(),
 	}
-	if err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil {
+	if _, err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil {
 		return &containers, fmt.Errorf("failed to render Quesma config: %v", err)
 	}
 
@@ -344,14 +355,39 @@ func setupAllContainersWithCh(ctx context.Context, quesmaConfigTemplate string)
 		"clickhouse_host":    GetInternalDockerHost(),
 		"clickhouse_port":    chPort.Port(),
 	}
-	if err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil {
+	configPath, err := RenderQuesmaConfig(quesmaConfigTemplate, data)
+	if err != nil {
 		return &containers, fmt.Errorf("failed to render Quesma config: %v", err)
 	}
 
-	quesma, err := setupQuesma(ctx, quesmaConfigTemplate)
-	containers.Quesma = &quesma
-	if err != nil {
-		return &containers, fmt.Errorf("failed to start Quesma, %v", err)
+	var quesma testcontainers.Container
+	if debugQuesmaDuringTestRun {
+		debuggerQuesmaConfig := filepath.Join(filepath.Dir(configPath), "quesma-with-debugger.yml")
+		content, err := os.ReadFile(configPath)
+		if err != nil {
+			return &containers, fmt.Errorf("failed to read rendered Quesma config: %v", err)
+		}
+		if err := os.WriteFile(debuggerQuesmaConfig, content, 0644); err != nil {
+			return &containers, fmt.Errorf("failed to write quesma-with-debugger.yml: %v", err)
+		}
+		log.Printf("Quesma config rendered to: %s", debuggerQuesmaConfig)
+
+		log.Printf("Waiting for you to start Quesma form your IDE using `Debug Quesma IT` configuration")
+		for {
+			if resp, err := http.Get("http://localhost:8080"); err == nil {
+				resp.Body.Close()
+				break
+			}
+			log.Printf("Waiting for Quesma HTTP server at port 8080...")
+			time.Sleep(1 * time.Second)
+			quesma = NewManuallyCreatedContainer()
+		}
+	} else {
+		quesma, err = setupQuesma(ctx, quesmaConfigTemplate)
+		if err != nil {
+			return &containers, fmt.Errorf("failed to start Quesma: %v", err)
+		}
+		containers.Quesma = &quesma
 	}
 
 	kibana, err := setupKibana(ctx, quesma)