Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add Docker Swarm support as a new deployment package #602

Closed
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions Dockerfile.docker-swarm-agent
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
FROM golang:1.24 AS builder
ARG TARGETOS
ARG TARGETARCH
ARG VERSION
ARG COMMIT

WORKDIR /workspace
COPY go.mod go.mod
COPY go.sum go.sum
RUN go mod download

COPY api/ api/
COPY cmd/agent/docker-swarm/ cmd/agent/docker-swarm/
# doesn't exist (yet?)
# COPY pkg/ pkg/
COPY internal/ internal/
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \
go build -a -o agent \
-ldflags="-s -w -X github.com/glasskube/distr/internal/buildconfig.version=${VERSION:-snapshot} -X github.com/glasskube/distr/internal/buildconfig.commit=${COMMIT}" \
./cmd/agent/docker-swarm/

FROM docker:27.3.1-alpine3.20
WORKDIR /
COPY --from=builder /workspace/agent .

ENTRYPOINT ["/agent"]
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ docker-build-docker-agent:
docker-build-kubernetes-agent:
docker build -f Dockerfile.kubernetes-agent --tag ghcr.io/glasskube/distr/kubernetes-agent:$(VERSION) --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) --network host .

.PHONY: docker-build-docker-swarm-agent
docker-build-docker-swarm-agent:
docker build -f Dockerfile.docker-swarm-agent --tag ghcr.io/glasskube/distr/docker-swarm-agent:$(VERSION) --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) --network host .

.PHONY: docker-build
docker-build: docker-build-hub docker-build-docker-agent docker-build-kubernetes-agent

Expand Down
99 changes: 99 additions & 0 deletions cmd/agent/docker-swarm/agent_deployment.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package main

import (
"encoding/json"
"errors"
"fmt"
"os"
"path"

"github.com/glasskube/distr/api"
"github.com/google/uuid"
)

type AgentDeployment struct {
ID uuid.UUID `json:"id"`
RevisionID uuid.UUID `json:"revisionId"`
ProjectName string `json:"projectName"`
}

func (d *AgentDeployment) FileName() string {
return path.Join(agentDeploymentDir(), d.ID.String())
}

func agentDeploymentDir() string {
return path.Join(ScratchDir(), "deployments")
}

func NewAgentDeployment(deployment api.DockerAgentDeployment) (*AgentDeployment, error) {
if name, err := getProjectName(deployment.ComposeFile); err != nil {
return nil, err
} else {
return &AgentDeployment{ID: deployment.ID, RevisionID: deployment.RevisionID, ProjectName: name}, nil
}
}

func getProjectName(data []byte) (string, error) {
if compose, err := DecodeComposeFile(data); err != nil {
return "", err
} else if name, ok := compose["name"].(string); !ok {
return "", fmt.Errorf("name is not a string")
} else {
return name, nil
}
}

func GetExistingDeployments() ([]AgentDeployment, error) {
if entries, err := os.ReadDir(agentDeploymentDir()); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, err
} else {
fn := func(name string) (*AgentDeployment, error) {
if file, err := os.Open(path.Join(agentDeploymentDir(), name)); err != nil {
return nil, err
} else {
defer file.Close()
var d AgentDeployment
if err := json.NewDecoder(file).Decode(&d); err != nil {
return nil, err
}
return &d, nil
}
}
result := make([]AgentDeployment, 0, len(entries))
for _, entry := range entries {
if !entry.IsDir() {
if d, err := fn(entry.Name()); err != nil {
return nil, err
} else {
result = append(result, *d)
}
}
}
return result, nil
}
}

func SaveDeployment(deployment AgentDeployment) error {
if err := os.MkdirAll(path.Dir(deployment.FileName()), 0o700); err != nil {
return err
}

file, err := os.Create(deployment.FileName())
if err != nil {
return err
}
defer file.Close()

if err := json.NewEncoder(file).Encode(deployment); err != nil {
return err
}

return nil
}

func DeleteDeployment(deployment AgentDeployment) error {
return os.Remove(deployment.FileName())
}
10 changes: 10 additions & 0 deletions cmd/agent/docker-swarm/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package main

import "os"

func ScratchDir() string {
if dir := os.Getenv("DISTR_AGENT_SCRATCH_DIR"); dir != "" {
return dir
}
return "./scratch"
}
121 changes: 121 additions & 0 deletions cmd/agent/docker-swarm/docker_actions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package main

import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"os"
"os/exec"
"strings"

"github.com/glasskube/distr/api"
"github.com/glasskube/distr/internal/agentauth"
"go.uber.org/zap"
)

func cleanComposeFile(composeData []byte) []byte {
lines := strings.Split(string(composeData), "\n")
cleanedLines := make([]string, 0, 50)

for _, line := range lines {
// Skip lines that define `name:`
if strings.HasPrefix(strings.TrimSpace(line), "name:") {
continue
}
cleanedLines = append(cleanedLines, line)
}
return []byte(strings.Join(cleanedLines, "\n"))
}
func parseEnvFile(envData []byte) (map[string]string, error) {
envVars := make(map[string]string)
scanner := bufio.NewScanner(bytes.NewReader(envData))
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") {
continue // Skip empty lines and comments
}
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid environment variable: %s", line)
}
envVars[parts[0]] = parts[1]
}
return envVars, scanner.Err()
}
func replaceEnvVars(composeData []byte, envVars map[string]string) []byte {
content := string(composeData)
for key, value := range envVars {
placeholder := fmt.Sprintf("${%s}", key)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Parsing the env file and replacing variables like this seems kind of brittle. I think understand that docker stack doesn't support variable substitution directly, but a better way appears to be using docker compose config to perform variable substitution and other generators and run docker stack deploy with the output of that (source). Do you think that would work?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, let me look into this.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I previously used this script, and it was incredibly helpful for managing env files with Docker Swarm. I just implemented it.

start_with_env_file() {
    
    stack=${1:-${PWD##*/}} # by default, the name of the cointaining folder
    compose_file=${2:-docker-compose.yml}

    if [ ! -f $compose_file ]; then
        echo "Misses compose file: $compose_file" >&2
        return 1
    fi

    # execute as a subcommand in order to avoid the variables remain set
    (
        # export variables excluding comments
        [ -f .env ] && export $(sed '/^#/d' .env)

        # Use dsd your_stack your_compose_file to override the defaults
        docker stack deploy --compose-file $compose_file "home"
    )
}

content = strings.ReplaceAll(content, placeholder, value)
}
return []byte(content)
}

func ApplyComposeFileSwarm(
ctx context.Context,
deployment api.DockerAgentDeployment,
) (*AgentDeployment, string, error) {
agentDeployment, err := NewAgentDeployment(deployment)
if err != nil {
return nil, "", err
}

// Process environment variables
envVars := make(map[string]string)
if deployment.EnvFile != nil {
envVars, err = parseEnvFile(deployment.EnvFile)
if err != nil {
logger.Error("failed to parse env file", zap.Error(err))
return nil, "", fmt.Errorf("failed to parse env file: %w", err)
}
}

// Ensure Docker Swarm is initialized
initCmd := exec.CommandContext(ctx, "docker", "info", "--format", "'{{.Swarm.LocalNodeState}}'")
initOutput, _ := initCmd.CombinedOutput()
if !strings.Contains(string(initOutput), "active") {
logger.Error("docker swarm not initializ: ", zap.String("output", string(initOutput)), zap.Error(err))
return nil, "", fmt.Errorf("docker swarm not initialize: %s ", string(initOutput))

}

// fix: Clean up Compose file: remove `name` field and inject environment variables
cleanedCompose := cleanComposeFile(deployment.ComposeFile)
finalCompose := replaceEnvVars(cleanedCompose, envVars)

// Run `docker stack deploy`
composeArgs := []string{"stack", "deploy", "-c", "-", agentDeployment.ProjectName}
cmd := exec.CommandContext(ctx, "docker", composeArgs...)
cmd.Stdin = bytes.NewReader(finalCompose)
cmd.Env = append(os.Environ(), agentauth.DockerConfigEnv(deployment.AgentDeployment)...)

cmdOut, err := cmd.CombinedOutput()
statusStr := string(cmdOut)
logger.Debug("docker stack deploy returned", zap.String("output", statusStr), zap.Error(err))

if err != nil {
return nil, "", errors.New(statusStr)
}

return agentDeployment, statusStr, nil
}

func UninstallDockerSwarm(ctx context.Context, deployment AgentDeployment) error {

cmd := exec.CommandContext(ctx, "docker", "stack", "rm", deployment.ProjectName)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to remove Docker Swarm stack: %w: %v", err, string(out))
}

// Optional: Prune unused networks created by Swarm
pruneCmd := exec.CommandContext(ctx, "docker", "network", "prune", "-f")
pruneOut, pruneErr := pruneCmd.CombinedOutput()
if pruneErr != nil {
logger.Warn("Failed to prune networks", zap.String("output", string(pruneOut)), zap.Error(pruneErr))
}

return nil
}
109 changes: 109 additions & 0 deletions cmd/agent/docker-swarm/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package main

import (
"context"
"os"
"os/signal"
"syscall"
"time"

"github.com/glasskube/distr/internal/agentauth"
"github.com/glasskube/distr/internal/agentclient"
"github.com/glasskube/distr/internal/util"
"go.uber.org/multierr"
"go.uber.org/zap"
)

var (
interval = 5 * time.Second
logger = util.Require(zap.NewDevelopment())
client = util.Require(agentclient.NewFromEnv(logger))
agentVersionID = os.Getenv("DISTR_AGENT_VERSION_ID")
)

func init() {
if intervalStr, ok := os.LookupEnv("DISTR_INTERVAL"); ok {
interval = util.Require(time.ParseDuration(intervalStr))
}
if agentVersionID == "" {
logger.Warn("DISTR_AGENT_VERSION_ID is not set. self updates will be disabled")
}
}

func main() {
ctx, cancel := context.WithCancel(context.Background())
go func() {
sigint := make(chan os.Signal, 1)
signal.Notify(sigint, syscall.SIGTERM, syscall.SIGINT)
<-sigint
logger.Info("received termination signal")
cancel()
}()
tick := time.Tick(interval)
loop:
for ctx.Err() == nil {
select {
case <-tick:
case <-ctx.Done():
break loop
}

if resource, err := client.DockerResource(ctx); err != nil {
logger.Error("failed to get resource", zap.Error(err))
} else {
if agentVersionID != "" {
if agentVersionID != resource.Version.ID.String() {
logger.Info("agent version has changed. starting self-update")
if err := RunAgentSelfUpdate(ctx); err != nil {
logger.Error("self update failed", zap.Error(err))
// TODO: Support status without revision ID?
if resource.Deployment != nil {
if err := client.Status(ctx, resource.Deployment.RevisionID, "", err); err != nil {
logger.Error("failed to send status", zap.Error(err))
}
}
} else {
logger.Info("self-update has been applied")
continue
}
} else {
logger.Debug("agent version is up to date")
}
}

if deployments, err := GetExistingDeployments(); err != nil {
logger.Error("could not get existing deployments", zap.Error(err))
} else {
for _, deployment := range deployments {
if resource.Deployment == nil || resource.Deployment.ID != deployment.ID {
logger.Info("uninstalling old deployment", zap.String("id", deployment.ID.String()))
if err := UninstallDockerSwarm(ctx, deployment); err != nil {
logger.Error("could not uninstall deployment", zap.Error(err))
} else if err := DeleteDeployment(deployment); err != nil {
logger.Error("could not delete deployment", zap.Error(err))
}
}
}
}

if resource.Deployment == nil {
logger.Info("no deployment in resource response")
continue
}

var agentDeployment *AgentDeployment
var status string
_, err = agentauth.EnsureAuth(ctx, resource.Deployment.AgentDeployment)
if err != nil {
logger.Error("docker auth error", zap.Error(err))
} else if agentDeployment, status, err = ApplyComposeFileSwarm(ctx, *resource.Deployment); err == nil {
multierr.AppendInto(&err, SaveDeployment(*agentDeployment))
}

if statusErr := client.Status(ctx, resource.Deployment.RevisionID, status, err); statusErr != nil {
logger.Error("failed to send status", zap.Error(statusErr))
}
}
}
logger.Info("shutting down")
}
Loading