-
Notifications
You must be signed in to change notification settings - Fork 17
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add Docker Swarm support as a new deployment package #602
Closed
asherAbecasiss
wants to merge
3
commits into
glasskube:main
from
asherAbecasiss:docker-swarm-feature
Closed
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
FROM golang:1.24 AS builder | ||
ARG TARGETOS | ||
ARG TARGETARCH | ||
ARG VERSION | ||
ARG COMMIT | ||
|
||
WORKDIR /workspace | ||
COPY go.mod go.mod | ||
COPY go.sum go.sum | ||
RUN go mod download | ||
|
||
COPY api/ api/ | ||
COPY cmd/agent/docker-swarm/ cmd/agent/docker-swarm/ | ||
# doesn't exist (yet?) | ||
# COPY pkg/ pkg/ | ||
COPY internal/ internal/ | ||
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ | ||
go build -a -o agent \ | ||
-ldflags="-s -w -X github.com/glasskube/distr/internal/buildconfig.version=${VERSION:-snapshot} -X github.com/glasskube/distr/internal/buildconfig.commit=${COMMIT}" \ | ||
./cmd/agent/docker-swarm/ | ||
|
||
FROM docker:27.3.1-alpine3.20 | ||
WORKDIR / | ||
COPY --from=builder /workspace/agent . | ||
|
||
ENTRYPOINT ["/agent"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
package main | ||
|
||
import ( | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
"os" | ||
"path" | ||
|
||
"github.com/glasskube/distr/api" | ||
"github.com/google/uuid" | ||
) | ||
|
||
type AgentDeployment struct { | ||
ID uuid.UUID `json:"id"` | ||
RevisionID uuid.UUID `json:"revisionId"` | ||
ProjectName string `json:"projectName"` | ||
} | ||
|
||
func (d *AgentDeployment) FileName() string { | ||
return path.Join(agentDeploymentDir(), d.ID.String()) | ||
} | ||
|
||
func agentDeploymentDir() string { | ||
return path.Join(ScratchDir(), "deployments") | ||
} | ||
|
||
func NewAgentDeployment(deployment api.DockerAgentDeployment) (*AgentDeployment, error) { | ||
if name, err := getProjectName(deployment.ComposeFile); err != nil { | ||
return nil, err | ||
} else { | ||
return &AgentDeployment{ID: deployment.ID, RevisionID: deployment.RevisionID, ProjectName: name}, nil | ||
} | ||
} | ||
|
||
func getProjectName(data []byte) (string, error) { | ||
if compose, err := DecodeComposeFile(data); err != nil { | ||
return "", err | ||
} else if name, ok := compose["name"].(string); !ok { | ||
return "", fmt.Errorf("name is not a string") | ||
} else { | ||
return name, nil | ||
} | ||
} | ||
|
||
func GetExistingDeployments() ([]AgentDeployment, error) { | ||
if entries, err := os.ReadDir(agentDeploymentDir()); err != nil { | ||
if errors.Is(err, os.ErrNotExist) { | ||
return nil, nil | ||
} | ||
return nil, err | ||
} else { | ||
fn := func(name string) (*AgentDeployment, error) { | ||
if file, err := os.Open(path.Join(agentDeploymentDir(), name)); err != nil { | ||
return nil, err | ||
} else { | ||
defer file.Close() | ||
var d AgentDeployment | ||
if err := json.NewDecoder(file).Decode(&d); err != nil { | ||
return nil, err | ||
} | ||
return &d, nil | ||
} | ||
} | ||
result := make([]AgentDeployment, 0, len(entries)) | ||
for _, entry := range entries { | ||
if !entry.IsDir() { | ||
if d, err := fn(entry.Name()); err != nil { | ||
return nil, err | ||
} else { | ||
result = append(result, *d) | ||
} | ||
} | ||
} | ||
return result, nil | ||
} | ||
} | ||
|
||
func SaveDeployment(deployment AgentDeployment) error { | ||
if err := os.MkdirAll(path.Dir(deployment.FileName()), 0o700); err != nil { | ||
return err | ||
} | ||
|
||
file, err := os.Create(deployment.FileName()) | ||
if err != nil { | ||
return err | ||
} | ||
defer file.Close() | ||
|
||
if err := json.NewEncoder(file).Encode(deployment); err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func DeleteDeployment(deployment AgentDeployment) error { | ||
return os.Remove(deployment.FileName()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
package main | ||
|
||
import "os" | ||
|
||
func ScratchDir() string { | ||
if dir := os.Getenv("DISTR_AGENT_SCRATCH_DIR"); dir != "" { | ||
return dir | ||
} | ||
return "./scratch" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
package main | ||
|
||
import ( | ||
"bufio" | ||
"bytes" | ||
"context" | ||
"errors" | ||
"fmt" | ||
"os" | ||
"os/exec" | ||
"strings" | ||
|
||
"github.com/glasskube/distr/api" | ||
"github.com/glasskube/distr/internal/agentauth" | ||
"go.uber.org/zap" | ||
) | ||
|
||
func cleanComposeFile(composeData []byte) []byte { | ||
lines := strings.Split(string(composeData), "\n") | ||
cleanedLines := make([]string, 0, 50) | ||
|
||
for _, line := range lines { | ||
// Skip lines that define `name:` | ||
if strings.HasPrefix(strings.TrimSpace(line), "name:") { | ||
continue | ||
} | ||
cleanedLines = append(cleanedLines, line) | ||
} | ||
return []byte(strings.Join(cleanedLines, "\n")) | ||
} | ||
func parseEnvFile(envData []byte) (map[string]string, error) { | ||
envVars := make(map[string]string) | ||
scanner := bufio.NewScanner(bytes.NewReader(envData)) | ||
for scanner.Scan() { | ||
line := scanner.Text() | ||
if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { | ||
continue // Skip empty lines and comments | ||
} | ||
parts := strings.SplitN(line, "=", 2) | ||
if len(parts) != 2 { | ||
return nil, fmt.Errorf("invalid environment variable: %s", line) | ||
} | ||
envVars[parts[0]] = parts[1] | ||
} | ||
return envVars, scanner.Err() | ||
} | ||
func replaceEnvVars(composeData []byte, envVars map[string]string) []byte { | ||
content := string(composeData) | ||
for key, value := range envVars { | ||
placeholder := fmt.Sprintf("${%s}", key) | ||
content = strings.ReplaceAll(content, placeholder, value) | ||
} | ||
return []byte(content) | ||
} | ||
|
||
func ApplyComposeFileSwarm( | ||
ctx context.Context, | ||
deployment api.DockerAgentDeployment, | ||
) (*AgentDeployment, string, error) { | ||
agentDeployment, err := NewAgentDeployment(deployment) | ||
if err != nil { | ||
return nil, "", err | ||
} | ||
|
||
// Process environment variables | ||
envVars := make(map[string]string) | ||
if deployment.EnvFile != nil { | ||
envVars, err = parseEnvFile(deployment.EnvFile) | ||
if err != nil { | ||
logger.Error("failed to parse env file", zap.Error(err)) | ||
return nil, "", fmt.Errorf("failed to parse env file: %w", err) | ||
} | ||
} | ||
|
||
// Ensure Docker Swarm is initialized | ||
initCmd := exec.CommandContext(ctx, "docker", "info", "--format", "'{{.Swarm.LocalNodeState}}'") | ||
initOutput, _ := initCmd.CombinedOutput() | ||
if !strings.Contains(string(initOutput), "active") { | ||
logger.Error("docker swarm not initializ: ", zap.String("output", string(initOutput)), zap.Error(err)) | ||
return nil, "", fmt.Errorf("docker swarm not initialize: %s ", string(initOutput)) | ||
|
||
} | ||
|
||
// fix: Clean up Compose file: remove `name` field and inject environment variables | ||
cleanedCompose := cleanComposeFile(deployment.ComposeFile) | ||
finalCompose := replaceEnvVars(cleanedCompose, envVars) | ||
|
||
// Run `docker stack deploy` | ||
composeArgs := []string{"stack", "deploy", "-c", "-", agentDeployment.ProjectName} | ||
cmd := exec.CommandContext(ctx, "docker", composeArgs...) | ||
cmd.Stdin = bytes.NewReader(finalCompose) | ||
cmd.Env = append(os.Environ(), agentauth.DockerConfigEnv(deployment.AgentDeployment)...) | ||
|
||
cmdOut, err := cmd.CombinedOutput() | ||
statusStr := string(cmdOut) | ||
logger.Debug("docker stack deploy returned", zap.String("output", statusStr), zap.Error(err)) | ||
|
||
if err != nil { | ||
return nil, "", errors.New(statusStr) | ||
} | ||
|
||
return agentDeployment, statusStr, nil | ||
} | ||
|
||
func UninstallDockerSwarm(ctx context.Context, deployment AgentDeployment) error { | ||
|
||
cmd := exec.CommandContext(ctx, "docker", "stack", "rm", deployment.ProjectName) | ||
out, err := cmd.CombinedOutput() | ||
if err != nil { | ||
return fmt.Errorf("failed to remove Docker Swarm stack: %w: %v", err, string(out)) | ||
} | ||
|
||
// Optional: Prune unused networks created by Swarm | ||
pruneCmd := exec.CommandContext(ctx, "docker", "network", "prune", "-f") | ||
pruneOut, pruneErr := pruneCmd.CombinedOutput() | ||
if pruneErr != nil { | ||
logger.Warn("Failed to prune networks", zap.String("output", string(pruneOut)), zap.Error(pruneErr)) | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"os" | ||
"os/signal" | ||
"syscall" | ||
"time" | ||
|
||
"github.com/glasskube/distr/internal/agentauth" | ||
"github.com/glasskube/distr/internal/agentclient" | ||
"github.com/glasskube/distr/internal/util" | ||
"go.uber.org/multierr" | ||
"go.uber.org/zap" | ||
) | ||
|
||
var ( | ||
interval = 5 * time.Second | ||
logger = util.Require(zap.NewDevelopment()) | ||
client = util.Require(agentclient.NewFromEnv(logger)) | ||
agentVersionID = os.Getenv("DISTR_AGENT_VERSION_ID") | ||
) | ||
|
||
func init() { | ||
if intervalStr, ok := os.LookupEnv("DISTR_INTERVAL"); ok { | ||
interval = util.Require(time.ParseDuration(intervalStr)) | ||
} | ||
if agentVersionID == "" { | ||
logger.Warn("DISTR_AGENT_VERSION_ID is not set. self updates will be disabled") | ||
} | ||
} | ||
|
||
func main() { | ||
ctx, cancel := context.WithCancel(context.Background()) | ||
go func() { | ||
sigint := make(chan os.Signal, 1) | ||
signal.Notify(sigint, syscall.SIGTERM, syscall.SIGINT) | ||
<-sigint | ||
logger.Info("received termination signal") | ||
cancel() | ||
}() | ||
tick := time.Tick(interval) | ||
loop: | ||
for ctx.Err() == nil { | ||
select { | ||
case <-tick: | ||
case <-ctx.Done(): | ||
break loop | ||
} | ||
|
||
if resource, err := client.DockerResource(ctx); err != nil { | ||
logger.Error("failed to get resource", zap.Error(err)) | ||
} else { | ||
if agentVersionID != "" { | ||
if agentVersionID != resource.Version.ID.String() { | ||
logger.Info("agent version has changed. starting self-update") | ||
if err := RunAgentSelfUpdate(ctx); err != nil { | ||
logger.Error("self update failed", zap.Error(err)) | ||
// TODO: Support status without revision ID? | ||
if resource.Deployment != nil { | ||
if err := client.Status(ctx, resource.Deployment.RevisionID, "", err); err != nil { | ||
logger.Error("failed to send status", zap.Error(err)) | ||
} | ||
} | ||
} else { | ||
logger.Info("self-update has been applied") | ||
continue | ||
} | ||
} else { | ||
logger.Debug("agent version is up to date") | ||
} | ||
} | ||
|
||
if deployments, err := GetExistingDeployments(); err != nil { | ||
logger.Error("could not get existing deployments", zap.Error(err)) | ||
} else { | ||
for _, deployment := range deployments { | ||
if resource.Deployment == nil || resource.Deployment.ID != deployment.ID { | ||
logger.Info("uninstalling old deployment", zap.String("id", deployment.ID.String())) | ||
if err := UninstallDockerSwarm(ctx, deployment); err != nil { | ||
logger.Error("could not uninstall deployment", zap.Error(err)) | ||
} else if err := DeleteDeployment(deployment); err != nil { | ||
logger.Error("could not delete deployment", zap.Error(err)) | ||
} | ||
} | ||
} | ||
} | ||
|
||
if resource.Deployment == nil { | ||
logger.Info("no deployment in resource response") | ||
continue | ||
} | ||
|
||
var agentDeployment *AgentDeployment | ||
var status string | ||
_, err = agentauth.EnsureAuth(ctx, resource.Deployment.AgentDeployment) | ||
if err != nil { | ||
logger.Error("docker auth error", zap.Error(err)) | ||
} else if agentDeployment, status, err = ApplyComposeFileSwarm(ctx, *resource.Deployment); err == nil { | ||
multierr.AppendInto(&err, SaveDeployment(*agentDeployment)) | ||
} | ||
|
||
if statusErr := client.Status(ctx, resource.Deployment.RevisionID, status, err); statusErr != nil { | ||
logger.Error("failed to send status", zap.Error(statusErr)) | ||
} | ||
} | ||
} | ||
logger.Info("shutting down") | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Parsing the env file and replacing variables like this seems kind of brittle. I think understand that
docker stack
doesn't support variable substitution directly, but a better way appears to be usingdocker compose config
to perform variable substitution and other generators and rundocker stack deploy
with the output of that (source). Do you think that would work?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sure, let me look into this.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I previously used this script, and it was incredibly helpful for managing env files with Docker Swarm. I just implemented it.