Merge branch 'develop' of github.com:FactorT/semaphore into fix-PAASCOMMON-39

Author: Bazhin Maksim Sergeevich

CONTRIBUTING.md

@@ -43,7 +43,7 @@ echo "create database semaphore;" | mysql -uroot -p
 5) Compile, set up & run
 
 ```
-task compile
+task build
 go run cli/main.go setup
 go run cli/main.go service --config ./config.json
 ```
@@ -64,7 +64,7 @@ As Dredd and the application database config may differ it expects it's own conf
 
 1) Build Dredd hooks:
     ````bash
-    task compile:api:hooks
+    task e2e:hooks
     ```
 2) Install Dredd globally
     ```bash

README.md

@@ -33,12 +33,12 @@ Semaphore UI allows you to:
 
 ## Key Concepts
 
-1. **Projects** is a collection of related resources, configurations, and tasks. Each project allows you to organize and manage your automation efforts in one place, defining the scope of tasks such as deploying applications, running scripts, or orchestrating cloud resources. Projects help group resources, inventories, task templates, and environments for streamlined automation workflows.
-2. **Task Templates** are reusable definitions of tasks that can be executed on demand or scheduled. A template specifies what actions should be performed, such as running Ansible playbooks, Terraform configurations, or other automation tasks. By using templates, you can standardize tasks and easily re-execute them with minimal effort, ensuring consistent results across different environments.
-3. **Task** is a specific instance of a job or operation executed by Semaphore. It refers to running a predefined action (like an Ansible playbook or a script) using a task template. Tasks can be initiated manually or automatically through schedules and are tracked to give you detailed feedback on the execution, including success, failure, and logs.
-4. **Schedules** allow you to automate task execution at specified times or intervals. This feature is useful for running periodic maintenance tasks, backups, or deployments without manual intervention. You can configure recurring schedules to ensure important automation tasks are performed regularly and on time.
-5. **Inventory** is a collection of target hosts (servers, virtual machines, containers, etc.) on which tasks will be executed. The inventory includes details about the managed nodes such as IP addresses, SSH credentials, and grouping information. It allows for dynamic control over which environments and hosts your automation will interact with.
-6. **Environment** refers to a configuration context that holds sensitive information such as environment variables and secrets used by tasks during execution. It separates sensitive data from task templates and allows you to switch between different setups while running the same task template across different environments securely.
+1. **Projects** is a collection of related resources, configurations, and tasks.
+2. **Task Templates** are reusable definitions of tasks that can be executed on demand or scheduled.
+3. **Task** is a specific instance of a job or operation executed by Semaphore.
+4. **Schedules** allow you to automate task execution at specified times or intervals.
+5. **Inventory** is a collection of target hosts (servers, virtual machines, containers, etc.) on which tasks will be executed.
+6. **Variable Group** refers to a configuration context that holds sensitive information such as environment variables and secrets used by tasks during execution.
 
 ## Getting Started
 
@@ -70,14 +70,6 @@ We recommend using the [Container Configurator](https://semaphoreui.com/install/
 
 We offer a SaaS solution for using Semaphore UI without installation. Check it out at [Semaphore Cloud](https://portal.semaphoreui.com).
 
-### Deploy VM from Marketplace
-
-Supported cloud providers:
-* [Semaphore Run](https://cloud.semaphore.run/servers/new/semaphore)
-* [AWS](https://aws.amazon.com/marketplace/pp/prodview-5noeat2jipwca)
-* [Yandex Cloud](https://yandex.cloud/en-ru/marketplace/products/fastlix/semaphore)
-* DigitalOcean (coming soon)
-
 ### Other Installation Methods
 
 For more installation options, visit our [Installation page](https://semaphoreui.com/install).

api-docs.yml

@@ -715,6 +715,8 @@ definitions:
         type:
           - string
           - 'null'
+      message:
+        type: string
 
   TaskOutput:
     type: object
@@ -2335,6 +2337,8 @@ paths:
                 type: string
               git_branch:
                 type: string
+              message:
+                type: string
       responses:
         201:
           description: Task queued

api/projects/runners.go

@@ -12,7 +12,7 @@ import (
 
 func GetRunners(w http.ResponseWriter, r *http.Request) {
 	project := context.Get(r, "project").(db.Project)
-	runners, err := helpers.Store(r).GetRunners(project.ID, false)
+	runners, err := helpers.Store(r).GetRunners(project.ID, false, nil)
 
 	if err != nil {
 		panic(err)

api/router.go

@@ -141,7 +141,7 @@ func Route() *mux.Router {
 	adminAPI.Path("/options").HandlerFunc(getOptions).Methods("GET", "HEAD")
 	adminAPI.Path("/options").HandlerFunc(setOption).Methods("POST")
 
-	adminAPI.Path("/runners").HandlerFunc(getGlobalRunners).Methods("GET", "HEAD")
+	adminAPI.Path("/runners").HandlerFunc(getAllRunners).Methods("GET", "HEAD")
 	adminAPI.Path("/runners").HandlerFunc(addGlobalRunner).Methods("POST", "HEAD")
 
 	globalRunnersAPI := adminAPI.PathPrefix("/runners").Subrouter()
@@ -221,6 +221,8 @@ func Route() *mux.Router {
 	projectUserAPI.Path("/tasks").HandlerFunc(projects.GetAllTasks).Methods("GET", "HEAD")
 	projectUserAPI.HandleFunc("/tasks/last", projects.GetLastTasks).Methods("GET", "HEAD")
 
+	projectUserAPI.Path("/stats").HandlerFunc(projects.GetTaskStats).Methods("GET", "HEAD")
+
 	projectUserAPI.Path("/templates").HandlerFunc(projects.GetTemplates).Methods("GET", "HEAD")
 	projectUserAPI.Path("/templates").HandlerFunc(projects.AddTemplate).Methods("POST")
 

api/runners.go

@@ -12,8 +12,8 @@ import (
 	"github.com/gorilla/context"
 )
 
-func getGlobalRunners(w http.ResponseWriter, r *http.Request) {
-	runners, err := helpers.Store(r).GetGlobalRunners(false)
+func getAllRunners(w http.ResponseWriter, r *http.Request) {
+	runners, err := helpers.Store(r).GetAllRunners(false, false)
 
 	if err != nil {
 		panic(err)

db/Migration.go

@@ -81,6 +81,8 @@ func GetMigrations() []Migration {
 		{Version: "2.12.3"},
 		{Version: "2.12.4"},
 		{Version: "2.12.5"},
+		{Version: "2.12.15"},
+		{Version: "2.13.0"},
 	}
 }
 

db/Runner.go

@@ -16,6 +16,7 @@ type Runner struct {
 	MaxParallelTasks int    `db:"max_parallel_tasks" json:"max_parallel_tasks"`
 	Active           bool   `db:"active" json:"active"`
 	Name             string `db:"name" json:"name"`
+	Tag              string `db:"tag" json:"tag"`
 
 	PublicKey *string `db:"public_key" json:"-"`
 }

db/Store.go

@@ -348,11 +348,11 @@ type Store interface {
 	SetViewPositions(projectID int, viewPositions map[int]int) error
 
 	GetRunner(projectID int, runnerID int) (Runner, error)
-	GetRunners(projectID int, activeOnly bool) ([]Runner, error)
+	GetRunners(projectID int, activeOnly bool, tag *string) ([]Runner, error)
 	DeleteRunner(projectID int, runnerID int) error
 	GetGlobalRunnerByToken(token string) (Runner, error)
 	GetGlobalRunner(runnerID int) (Runner, error)
-	GetGlobalRunners(activeOnly bool) ([]Runner, error)
+	GetAllRunners(activeOnly bool, globalOnly bool) ([]Runner, error)
 	DeleteGlobalRunner(runnerID int) error
 	UpdateRunner(runner Runner) error
 	CreateRunner(runner Runner) (Runner, error)

db/Template.go

@@ -61,6 +61,15 @@ const (
 	SurveyVarEnum TemplateType = "enum"
 )
 
+type AnsibleTemplateParams struct {
+	AllowDebug             bool     `json:"allow_debug"`
+	AllowOverrideInventory bool     `json:"allow_override_inventory"`
+	AllowOverrideLimit     bool     `json:"allow_override_limit"`
+	Limit                  []string `json:"limit"`
+	Tags                   []string `json:"tags"`
+	SkipTags               []string `json:"skip_tags"`
+}
+
 type TerraformTemplateParams struct {
 	AllowDestroy     bool `json:"allow_destroy"`
 	AllowAutoApprove bool `json:"allow_auto_approve"`
@@ -135,6 +144,31 @@ type Template struct {
 	Tasks int `db:"tasks" json:"tasks" backup:"-"`
 
 	TaskParams MapStringAnyField `db:"task_params" json:"task_params"`
+
+	RunnerTag *string `db:"runner_tag" json:"runner_tag"`
+}
+
+func (tpl *Template) FillParams(target interface{}) error {
+	content, err := json.Marshal(tpl.TaskParams)
+	if err != nil {
+		return nil
+	}
+	err = json.Unmarshal(content, target)
+	return err
+}
+
+func (tpl *Template) CanOverrideInventory() (ok bool, err error) {
+	switch tpl.App {
+	case AppAnsible, "":
+		var params AnsibleTemplateParams
+		err = tpl.FillParams(&params)
+		if err != nil {
+			return
+		}
+		ok = params.AllowOverrideInventory
+	}
+
+	return
 }
 
 func (tpl *Template) Validate() error {

db/bolt/global_runner.go

@@ -36,12 +36,18 @@ func (d *BoltDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
 	return
 }
 
-func (d *BoltDb) GetGlobalRunners(activeOnly bool) (runners []db.Runner, err error) {
+func (d *BoltDb) GetAllRunners(activeOnly bool, globalOnly bool) (runners []db.Runner, err error) {
 	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, func(i interface{}) bool {
 		runner := i.(db.Runner)
+
+		if globalOnly && runner.ProjectID != nil {
+			return false
+		}
+
 		if activeOnly {
 			return runner.Active
 		}
+
 		return true
 	}, &runners)
 	return

db/bolt/runner.go

@@ -11,7 +11,7 @@ func (d *BoltDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err e
 	return
 }
 
-func (d *BoltDb) GetRunners(projectID int, activeOnly bool) (runners []db.Runner, err error) {
+func (d *BoltDb) GetRunners(projectID int, activeOnly bool, tag *string) (runners []db.Runner, err error) {
 	runners = make([]db.Runner, 0)
 	return
 }

db/sql/SqlDb.go

@@ -806,6 +806,10 @@ func (d *SqlDb) GetTaskStats(projectID int, templateID *int, unit db.TaskStatUni
 		q = q.Where("template_id=?", *templateID)
 	}
 
+	if filter.UserID != nil {
+		q = q.Where("user_id=?", *filter.UserID)
+	}
+
 	if filter.Start != nil {
 		q = q.Where("start>=?", *filter.Start)
 	}

db/sql/global_runner.go

@@ -33,8 +33,13 @@ func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
 	return
 }
 
-func (d *SqlDb) GetGlobalRunners(activeOnly bool) (runners []db.Runner, err error) {
+func (d *SqlDb) GetAllRunners(activeOnly bool, globalOnly bool) (runners []db.Runner, err error) {
 	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, func(builder squirrel.SelectBuilder) squirrel.SelectBuilder {
+
+		if globalOnly {
+			builder = builder.Where("project_id is null")
+		}
+
 		if activeOnly {
 			builder = builder.Where("active=?", activeOnly)
 		}
@@ -51,11 +56,12 @@ func (d *SqlDb) DeleteGlobalRunner(runnerID int) (err error) {
 
 func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
 	_, err = d.exec(
-		"update runner set name=?, active=?, webhook=?, max_parallel_tasks=? where id=?",
+		"update runner set name=?, active=?, webhook=?, max_parallel_tasks=?, tag=? where id=?",
 		runner.Name,
 		runner.Active,
 		runner.Webhook,
 		runner.MaxParallelTasks,
+		runner.Tag,
 		runner.ID)
 
 	return
@@ -66,14 +72,15 @@ func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error)
 
 	insertID, err := d.insert(
 		"id",
-		"insert into runner (project_id, token, webhook, max_parallel_tasks, name, active, public_key) values (?, ?, ?, ?, ?, ?, ?)",
+		"insert into runner (project_id, token, webhook, max_parallel_tasks, name, active, public_key, tag) values (?, ?, ?, ?, ?, ?, ?, ?)",
 		runner.ProjectID,
 		token,
 		runner.Webhook,
 		runner.MaxParallelTasks,
 		runner.Name,
 		runner.Active,
-		runner.PublicKey)
+		runner.PublicKey,
+		runner.Tag)
 
 	if err != nil {
 		return

db/sql/migrations/v2.12.15.sql

@@ -0,0 +1,2 @@
+alter table `project__schedule` change `last_commit_hash` `last_commit_hash` varchar(64);
+alter table `task` change `commit_hash` `commit_hash` varchar(64);

db/sql/migrations/v2.13.0.sql

@@ -0,0 +1,2 @@
+alter table `runner` add column tag varchar(200) not null default '';
+alter table `project__template` add column runner_tag varchar(50);

db/sql/runner.go

@@ -11,7 +11,7 @@ func (d *SqlDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err er
 	return
 }
 
-func (d *SqlDb) GetRunners(projectID int, activeOnly bool) (runners []db.Runner, err error) {
+func (d *SqlDb) GetRunners(projectID int, activeOnly bool, tag *string) (runners []db.Runner, err error) {
 	runners = make([]db.Runner, 0)
 	return
 }

db/sql/template.go

@@ -17,28 +17,40 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
 
 	insertID, err := d.insert(
 		"id",
-		"insert into project__template (project_id, inventory_id, repository_id, environment_id, "+
-			"name, playbook, arguments, allow_override_args_in_task, description, `type`, start_version,"+
-			"build_template_id, view_id, autorun, survey_vars, suppress_success_alerts, app, git_branch)"+
-			"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+		"insert into project__template ("+
+			"project_id, inventory_id, repository_id, environment_id, name, "+
+			"playbook, arguments, allow_override_args_in_task, description, `type`, "+
+			"start_version, build_template_id, view_id, autorun, survey_vars, "+
+			"suppress_success_alerts, app, git_branch, runner_tag, task_params)"+
+			"values ("+
+			"?, ?, ?, ?, ?, "+
+			"?, ?, ?, ?, ?, "+
+			"?, ?, ?, ?, ?, "+
+			"?, ?, ?, ?, ?)",
 		template.ProjectID,
 		template.InventoryID,
 		template.RepositoryID,
 		template.EnvironmentID,
 		template.Name,
+
 		template.Playbook,
 		template.Arguments,
 		template.AllowOverrideArgsInTask,
 		template.Description,
 		template.Type,
+
 		template.StartVersion,
 		template.BuildTemplateID,
 		template.ViewID,
 		template.Autorun,
 		db.ObjectToJSON(template.SurveyVars),
+
 		template.SuppressSuccessAlerts,
 		template.App,
-		template.GitBranch)
+		template.GitBranch,
+		template.RunnerTag,
+		template.TaskParams,
+	)
 
 	if err != nil {
 		return
@@ -85,7 +97,9 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
 		"survey_vars=?, "+
 		"suppress_success_alerts=?, "+
 		"app=?, "+
-		"`git_branch`=? "+
+		"`git_branch`=?, "+
+		"task_params=?, "+
+		"runner_tag=? "+
 		"where id=? and project_id=?",
 		template.InventoryID,
 		template.RepositoryID,
@@ -104,6 +118,8 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
 		template.SuppressSuccessAlerts,
 		template.App,
 		template.GitBranch,
+		template.TaskParams,
+		template.RunnerTag,
 		template.ID,
 		template.ProjectID,
 	)
@@ -149,6 +165,7 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
 		"pt.start_version",
 		"pt.`type`",
 		"pt.`tasks`",
+		"pt.runner_tag",
 		"(SELECT `id` FROM `task` WHERE template_id = pt.id ORDER BY `id` DESC LIMIT 1) last_task_id").
 		From("project__template pt")
 

deployment/docker/runner/Dockerfile

@@ -24,7 +24,7 @@ RUN --mount=type=cache,target=/go/pkg \
     task build GOOS=${TARGETOS} GOARCH=${TARGETARCH}
 
 
-ENV OPENTOFU_VERSION="1.7.0"
+ENV OPENTOFU_VERSION="1.9.0"
 ENV TERRAFORM_VERSION="1.10.3"
 #ENV PULUMI_VERSION="3.116.1"
 

deployment/docker/server/Dockerfile

@@ -24,7 +24,7 @@ RUN --mount=type=cache,target=/go/pkg \
     task build GOOS=${TARGETOS} GOARCH=${TARGETARCH}
 
 
-ENV OPENTOFU_VERSION="1.7.0"
+ENV OPENTOFU_VERSION="1.9.0"
 ENV TERRAFORM_VERSION="1.10.3"
 #ENV PULUMI_VERSION="3.116.1"
 #ENV POWERSHELL_VERSION="3.116.1"
@@ -92,6 +92,7 @@ RUN apk add --no-cache -U python3-dev build-base openssl-dev libffi-dev cargo &&
      chown -R semaphore:0 /opt/semaphore
 
 USER 1001
+EXPOSE 3000
 
 ENV VIRTUAL_ENV="$ANSIBLE_VENV_PATH"
 ENV PATH="$ANSIBLE_VENV_PATH/bin:$PATH"