diff --git a/.gitignore b/.gitignore index ae6e018ac..c9d87756c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ ci/it/configs/*.yml elasticsearch-data -.idea +.idea/* +!.idea/runConfigurations/ +!.idea/runConfigurations/*.xml +# User-specific IDEA files +.idea/workspace.xml +.idea/tasks.xml +.idea/usage.statistics.xml +.idea/dictionaries/ __pycache__ mitmproxy/requests/*.txt mitmproxy/requests/*.mitm diff --git a/.idea/runConfigurations/Debug_Quesma_ITs.xml b/.idea/runConfigurations/Debug_Quesma_ITs.xml new file mode 100644 index 000000000..b231ffbfa --- /dev/null +++ b/.idea/runConfigurations/Debug_Quesma_ITs.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ci/it/README.md b/ci/it/README.md new file mode 100644 index 000000000..24450d596 --- /dev/null +++ b/ci/it/README.md @@ -0,0 +1,19 @@ +Quesma Integration Tests +======================== + +This directory contains integration tests for Quesma. +These are simple, end-to-end tests that verify the functionality of Quesma using [Testcontainers library](https://testcontainers.com). + + + + +How to debug +============ + +There is a way to run these tests agains a local Quesma instance with debugger attached. + +1. Set up a breakpoint in Quesma codebase. +2. Change the `debugQuesmaDuringTestRun` flag to `true` in `ci/it/testcases/utils.go` +3. Start an integration test (either with CLI or in your IDE using play button next to the test declaration). + ...Test case execution will block and wait until you start Quesma manually in IDE in debug mode) +4. Start Quesma in Debug mode using `Debug Quesma ITs` Run Configuration in your IDE. \ No newline at end of file diff --git a/ci/it/testcases/base.go b/ci/it/testcases/base.go index 801986d9a..69a76aa0d 100644 --- a/ci/it/testcases/base.go +++ b/ci/it/testcases/base.go @@ -43,6 +43,9 @@ func (tc *IntegrationTestcaseBase) Cleanup(ctx context.Context, t *testing.T) { } func (tc *IntegrationTestcaseBase) getQuesmaEndpoint() string { + if debugQuesmaDuringTestRun { // If debug mode is enabled, return a hardcoded endpoint for Quesma + return "http://localhost:8080" + } ctx := context.Background() q := *tc.Containers.Quesma p, _ := q.MappedPort(ctx, "8080/tcp") diff --git a/ci/it/testcases/container_utils.go b/ci/it/testcases/container_utils.go new file mode 100644 index 000000000..7106a5599 --- /dev/null +++ b/ci/it/testcases/container_utils.go @@ -0,0 +1,132 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +package testcases + +import ( + "context" + "github.com/docker/docker/api/types" + "github.com/docker/go-connections/nat" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/exec" + "io" + "time" +) + +func NewManuallyCreatedContainer() *ManuallyCreatedContainer { + return &ManuallyCreatedContainer{} +} + +type ManuallyCreatedContainer struct{} + +func (c ManuallyCreatedContainer) GetContainerID() string { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Endpoint(ctx context.Context, s string) (string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) PortEndpoint(ctx context.Context, port nat.Port, s string) (string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Host(ctx context.Context) (string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Inspect(ctx context.Context) (*types.ContainerJSON, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) MappedPort(ctx context.Context, port nat.Port) (nat.Port, error) { + return "8080/tcp", nil +} + +func (c ManuallyCreatedContainer) Ports(ctx context.Context) (nat.PortMap, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) SessionID() string { + panic("implement me") +} + +func (c ManuallyCreatedContainer) IsRunning() bool { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Start(ctx context.Context) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Stop(ctx context.Context, duration *time.Duration) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Terminate(ctx context.Context) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Logs(ctx context.Context) (io.ReadCloser, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) FollowOutput(consumer testcontainers.LogConsumer) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) StartLogProducer(ctx context.Context, option ...testcontainers.LogProductionOption) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) StopLogProducer() error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Name(ctx context.Context) (string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) State(ctx context.Context) (*types.ContainerState, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Networks(ctx context.Context) ([]string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) NetworkAliases(ctx context.Context) (map[string][]string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) Exec(ctx context.Context, cmd []string, options ...exec.ProcessOption) (int, io.Reader, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) ContainerIP(ctx context.Context) (string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) ContainerIPs(ctx context.Context) ([]string, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error { + panic("implement me") +} + +func (c ManuallyCreatedContainer) CopyFileFromContainer(ctx context.Context, filePath string) (io.ReadCloser, error) { + panic("implement me") +} + +func (c ManuallyCreatedContainer) GetLogProductionErrorChannel() <-chan error { + panic("implement me") +} diff --git a/ci/it/testcases/utils.go b/ci/it/testcases/utils.go index 9f4eeea72..978cf5b26 100644 --- a/ci/it/testcases/utils.go +++ b/ci/it/testcases/utils.go @@ -13,6 +13,7 @@ import ( "github.com/testcontainers/testcontainers-go/wait" "io" "log" + "net/http" "os" "path/filepath" "strings" @@ -21,12 +22,21 @@ import ( "time" ) +const ( + // debugQuesmaDuringTestRun should be set to `true` if you would like to run Quesma in IDE with debugger on + // during the integration test run. + debugQuesmaDuringTestRun = false +) + const configTemplatesDir = "configs" func GetInternalDockerHost() string { if check := os.Getenv("EXECUTING_ON_GITHUB_CI"); check != "" { return "localhost-for-github-ci" } + if debugQuesmaDuringTestRun { + return "localhost" + } return "host.docker.internal" // `host.testcontainers.internal` doesn't work for Docker Desktop for Mac. } @@ -264,26 +274,27 @@ func setupClickHouse(ctx context.Context) (testcontainers.Container, error) { }) } -func RenderQuesmaConfig(configTemplate string, data map[string]string) error { +func RenderQuesmaConfig(configTemplate string, data map[string]string) (string, error) { absPath, err := filepath.Abs(filepath.Join(".", configTemplatesDir, configTemplate)) content, err := os.ReadFile(absPath) if err != nil { - return fmt.Errorf("error reading YAML file: %v", err) + return "", fmt.Errorf("error reading YAML file: %v", err) } tmpl, err := template.New("yamlTemplate").Parse(string(content)) if err != nil { - return fmt.Errorf("error creating template: %v", err) + return "", fmt.Errorf("error creating template: %v", err) } var renderedContent bytes.Buffer err = tmpl.Execute(&renderedContent, data) if err != nil { - return fmt.Errorf("error executing template: %v", err) + return "", fmt.Errorf("error executing template: %v", err) } - err = os.WriteFile(strings.TrimSuffix(absPath, ".template"), renderedContent.Bytes(), 0644) + configPath := strings.TrimSuffix(absPath, ".template") + err = os.WriteFile(configPath, renderedContent.Bytes(), 0644) if err != nil { - return fmt.Errorf("error writing rendered YAML file: %v", err) + return "", fmt.Errorf("error writing rendered YAML file: %v", err) } - return nil + return configPath, nil } func setupContainersForTransparentProxy(ctx context.Context, quesmaConfigTemplate string) (*Containers, error) { @@ -300,7 +311,7 @@ func setupContainersForTransparentProxy(ctx context.Context, quesmaConfigTemplat "elasticsearch_host": GetInternalDockerHost(), "elasticsearch_port": esPort.Port(), } - if err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil { + if _, err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil { return &containers, fmt.Errorf("failed to render Quesma config: %v", err) } @@ -344,14 +355,39 @@ func setupAllContainersWithCh(ctx context.Context, quesmaConfigTemplate string) "clickhouse_host": GetInternalDockerHost(), "clickhouse_port": chPort.Port(), } - if err := RenderQuesmaConfig(quesmaConfigTemplate, data); err != nil { + configPath, err := RenderQuesmaConfig(quesmaConfigTemplate, data) + if err != nil { return &containers, fmt.Errorf("failed to render Quesma config: %v", err) } - quesma, err := setupQuesma(ctx, quesmaConfigTemplate) - containers.Quesma = &quesma - if err != nil { - return &containers, fmt.Errorf("failed to start Quesma, %v", err) + var quesma testcontainers.Container + if debugQuesmaDuringTestRun { + debuggerQuesmaConfig := filepath.Join(filepath.Dir(configPath), "quesma-with-debugger.yml") + content, err := os.ReadFile(configPath) + if err != nil { + return &containers, fmt.Errorf("failed to read rendered Quesma config: %v", err) + } + if err := os.WriteFile(debuggerQuesmaConfig, content, 0644); err != nil { + return &containers, fmt.Errorf("failed to write quesma-with-debugger.yml: %v", err) + } + log.Printf("Quesma config rendered to: %s", debuggerQuesmaConfig) + + log.Printf("Waiting for you to start Quesma form your IDE using `Debug Quesma IT` configuration") + for { + if resp, err := http.Get("http://localhost:8080"); err == nil { + resp.Body.Close() + break + } + log.Printf("Waiting for Quesma HTTP server at port 8080...") + time.Sleep(1 * time.Second) + quesma = NewManuallyCreatedContainer() + } + } else { + quesma, err = setupQuesma(ctx, quesmaConfigTemplate) + if err != nil { + return &containers, fmt.Errorf("failed to start Quesma: %v", err) + } + containers.Quesma = &quesma } kibana, err := setupKibana(ctx, quesma)