Skip to content

Commit 08b560b

Browse files
fiftinfiftin
authored
Merge pull request #2907 from semaphoreui/change_dir_struct
change dir struct
2 parents 4a4b951 + 3f46c41 commit 08b560b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1028
-519
lines changed

‎.dredd/hooks/helpers.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func addTestRunnerUser() {
2323
Username: "ITU-" + uid,
2424
Name: "ITU-" + uid,
2525
Email: uid + "@semaphore.test",
26-
Created: db.GetParsedTime(time.Now()),
26+
Created: db.GetParsedTime(time.Now().UTC()),
2727
Admin: true,
2828
}
2929

@@ -148,7 +148,7 @@ func addProject() *db.Project {
148148
chat := "Test"
149149
project := db.Project{
150150
Name: "ITP-" + uid,
151-
Created: time.Now(),
151+
Created: time.Now().UTC(),
152152
AlertChat: &chat,
153153
}
154154
project, err := store.CreateProject(project)
@@ -167,7 +167,7 @@ func addProject() *db.Project {
167167
func addUser() *db.User {
168168
uid := getUUID()
169169
user := db.User{
170-
Created: time.Now(),
170+
Created: time.Now().UTC(),
171171
Username: "ITU-" + uid,
172172
Email: "test@semaphore." + uid,
173173
Name: "ITU-" + uid,
@@ -215,7 +215,7 @@ func addTask() *db.Task {
215215
TemplateID: templateID,
216216
Status: "testing",
217217
UserID: &userPathTestUser.ID,
218-
Created: db.GetParsedTime(time.Now()),
218+
Created: db.GetParsedTime(time.Now().UTC()),
219219
}
220220

221221
t, err := store.CreateTask(t, 0)
@@ -284,7 +284,7 @@ func addIntegrationMatcher() *db.IntegrationMatcher {
284284
func addToken(tok string, user int) {
285285
_, err := store.CreateAPIToken(db.APIToken{
286286
ID: tok,
287-
Created: time.Now(),
287+
Created: time.Now().UTC(),
288288
UserID: user,
289289
Expired: false,
290290
})

‎api/cache.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package api
2+
3+
import (
4+
"github.com/gorilla/context"
5+
"github.com/semaphoreui/semaphore/api/helpers"
6+
"github.com/semaphoreui/semaphore/db"
7+
"github.com/semaphoreui/semaphore/util"
8+
log "github.com/sirupsen/logrus"
9+
"net/http"
10+
)
11+
12+
func clearCache(w http.ResponseWriter, r *http.Request) {
13+
currentUser := context.Get(r, "user").(*db.User)
14+
15+
if !currentUser.Admin {
16+
helpers.WriteJSON(w, http.StatusForbidden, map[string]string{
17+
"error": "User must be admin",
18+
})
19+
return
20+
}
21+
22+
err := util.Config.ClearTmpDir()
23+
if err != nil {
24+
log.Error(err)
25+
helpers.WriteJSON(w, http.StatusInternalServerError, map[string]string{
26+
"error": "Can not clear cache",
27+
})
28+
return
29+
}
30+
31+
w.WriteHeader(http.StatusNoContent)
32+
}

‎api/login.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func tryFindLDAPUser(username, password string) (*db.User, error) {
128128

129129
ldapUser := db.User{
130130
Username: strings.ToLower(claims.username),
131-
Created: time.Now(),
131+
Created: time.Now().UTC(),
132132
Name: claims.name,
133133
Email: claims.email,
134134
External: true,
@@ -162,8 +162,8 @@ func createSession(w http.ResponseWriter, r *http.Request, user db.User) {
162162

163163
newSession, err := helpers.Store(r).CreateSession(db.Session{
164164
UserID: user.ID,
165-
Created: time.Now(),
166-
LastActive: time.Now(),
165+
Created: time.Now().UTC(),
166+
LastActive: time.Now().UTC(),
167167
IP: r.Header.Get("X-Real-IP"),
168168
UserAgent: r.Header.Get("user-agent"),
169169
Expired: false,

‎api/projects/project.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
package projects
22

33
import (
4+
"github.com/gorilla/context"
5+
"github.com/gorilla/mux"
46
"github.com/semaphoreui/semaphore/api/helpers"
57
"github.com/semaphoreui/semaphore/db"
6-
"github.com/gorilla/mux"
8+
"github.com/semaphoreui/semaphore/util"
9+
log "github.com/sirupsen/logrus"
710
"net/http"
8-
9-
"github.com/gorilla/context"
1011
)
1112

1213
// ProjectMiddleware ensures a project exists and loads it to the context
@@ -102,6 +103,18 @@ func UpdateProject(w http.ResponseWriter, r *http.Request) {
102103
w.WriteHeader(http.StatusNoContent)
103104
}
104105

106+
func ClearCache(w http.ResponseWriter, r *http.Request) {
107+
project := context.Get(r, "project").(db.Project)
108+
109+
err := util.Config.ClearProjectTmpDir(project.ID)
110+
if err != nil {
111+
helpers.WriteError(w, err)
112+
return
113+
}
114+
115+
w.WriteHeader(http.StatusNoContent)
116+
}
117+
105118
// DeleteProject removes a project from the database
106119
func DeleteProject(w http.ResponseWriter, r *http.Request) {
107120
project := context.Get(r, "project").(db.Project)
@@ -113,5 +126,10 @@ func DeleteProject(w http.ResponseWriter, r *http.Request) {
113126
return
114127
}
115128

129+
err = util.Config.ClearProjectTmpDir(project.ID)
130+
if err != nil {
131+
log.Error(err)
132+
}
133+
116134
w.WriteHeader(http.StatusNoContent)
117135
}

‎api/projects/runners.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,7 @@ func DeleteRunner(w http.ResponseWriter, r *http.Request) {
4646
func SetRunnerActive(w http.ResponseWriter, r *http.Request) {
4747
w.WriteHeader(http.StatusNotFound)
4848
}
49+
50+
func ClearRunnerCache(w http.ResponseWriter, r *http.Request) {
51+
w.WriteHeader(http.StatusNotFound)
52+
}

‎api/router.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,15 @@ func Route() *mux.Router {
144144
adminAPI.Path("/runners").HandlerFunc(getAllRunners).Methods("GET", "HEAD")
145145
adminAPI.Path("/runners").HandlerFunc(addGlobalRunner).Methods("POST", "HEAD")
146146

147+
adminAPI.Path("/cache").HandlerFunc(clearCache).Methods("DELETE", "HEAD")
148+
147149
globalRunnersAPI := adminAPI.PathPrefix("/runners").Subrouter()
148150
globalRunnersAPI.Use(globalRunnerMiddleware)
149151
globalRunnersAPI.Path("/{runner_id}").HandlerFunc(getGlobalRunner).Methods("GET", "HEAD")
150152
globalRunnersAPI.Path("/{runner_id}").HandlerFunc(updateGlobalRunner).Methods("PUT", "POST")
151153
globalRunnersAPI.Path("/{runner_id}/active").HandlerFunc(setGlobalRunnerActive).Methods("POST")
152154
globalRunnersAPI.Path("/{runner_id}").HandlerFunc(deleteGlobalRunner).Methods("DELETE")
155+
globalRunnersAPI.Path("/{runner_id}/cache").HandlerFunc(clearGlobalRunnerCache).Methods("DELETE")
153156

154157
appsAPI := adminAPI.PathPrefix("/apps").Subrouter()
155158
appsAPI.Use(appMiddleware)
@@ -247,6 +250,7 @@ func Route() *mux.Router {
247250
projectRunnersAPI.Path("/{runner_id}").HandlerFunc(projects.UpdateRunner).Methods("PUT", "POST")
248251
projectRunnersAPI.Path("/{runner_id}/active").HandlerFunc(projects.SetRunnerActive).Methods("POST")
249252
projectRunnersAPI.Path("/{runner_id}").HandlerFunc(projects.DeleteRunner).Methods("DELETE")
253+
projectRunnersAPI.Path("/{runner_id}/cache").HandlerFunc(projects.ClearRunnerCache).Methods("DELETE")
250254

251255
//
252256
// Updating and deleting project
@@ -259,6 +263,10 @@ func Route() *mux.Router {
259263
meAPI.Use(projects.ProjectMiddleware)
260264
meAPI.HandleFunc("", projects.LeftProject).Methods("DELETE")
261265

266+
cacheAPI := authenticatedAPI.Path("/project/{project_id}/cache").Subrouter()
267+
cacheAPI.Use(projects.ProjectMiddleware)
268+
cacheAPI.HandleFunc("", projects.ClearCache).Methods("DELETE")
269+
262270
//
263271
// Manage project users
264272
projectAdminUsersAPI := authenticatedAPI.PathPrefix("/project/{project_id}").Subrouter()

‎api/runners.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,21 @@ func updateGlobalRunner(w http.ResponseWriter, r *http.Request) {
136136
w.WriteHeader(http.StatusNoContent)
137137
}
138138

139+
func clearGlobalRunnerCache(w http.ResponseWriter, r *http.Request) {
140+
runner := context.Get(r, "runner").(*db.Runner)
141+
142+
store := helpers.Store(r)
143+
144+
err := store.ClearRunnerCache(*runner)
145+
146+
if err != nil {
147+
helpers.WriteError(w, err)
148+
return
149+
}
150+
151+
w.WriteHeader(http.StatusNoContent)
152+
}
153+
139154
func deleteGlobalRunner(w http.ResponseWriter, r *http.Request) {
140155
runner := context.Get(r, "runner").(*db.Runner)
141156

‎api/runners/runners.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/semaphoreui/semaphore/pkg/task_logger"
1515
"github.com/semaphoreui/semaphore/services/runners"
1616
"github.com/semaphoreui/semaphore/util"
17+
log "github.com/sirupsen/logrus"
1718
"net/http"
1819
)
1920

@@ -94,8 +95,26 @@ func chunkRSAEncrypt(pub *rsa.PublicKey, plaintext []byte) ([]byte, error) {
9495
func GetRunner(w http.ResponseWriter, r *http.Request) {
9596
runner := context.Get(r, "runner").(db.Runner)
9697

98+
clearCache := false
99+
100+
err := helpers.Store(r).TouchRunner(runner)
101+
if err != nil {
102+
log.Error(err)
103+
helpers.WriteError(w, err)
104+
return
105+
}
106+
107+
if runner.CleaningRequested != nil && (runner.Touched == nil || runner.CleaningRequested.After(*runner.Touched)) {
108+
clearCache = true
109+
}
110+
97111
data := runners.RunnerState{
98112
AccessKeys: make(map[int]db.AccessKey),
113+
ClearCache: clearCache,
114+
}
115+
116+
if clearCache {
117+
data.CacheCleanProjectID = runner.ProjectID
99118
}
100119

101120
tasks := helpers.TaskPool(r).GetRunningTasks()

‎db/AccessKey.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ func (key *AccessKeyInstallation) Destroy() error {
102102
}
103103

104104
func (key *AccessKey) startSSHAgent(logger task_logger.Logger) (ssh.Agent, error) {
105+
106+
socketFilename := fmt.Sprintf("ssh-agent-%d-%s.sock", key.ID, random.String(10))
107+
108+
var socketFile string
109+
110+
if key.ProjectID == nil {
111+
socketFile = path.Join(util.Config.TmpPath, socketFilename)
112+
} else {
113+
socketFile = path.Join(util.Config.GetProjectTmpDir(*key.ProjectID), socketFilename)
114+
}
115+
105116
sshAgent := ssh.Agent{
106117
Logger: logger,
107118
Keys: []ssh.AgentKey{
@@ -110,7 +121,7 @@ func (key *AccessKey) startSSHAgent(logger task_logger.Logger) (ssh.Agent, error
110121
Passphrase: []byte(key.SshKey.Passphrase),
111122
},
112123
},
113-
SocketFile: path.Join(util.Config.TmpPath, fmt.Sprintf("ssh-agent-%d-%s.sock", key.ID, random.String(10))),
124+
SocketFile: socketFile,
114125
}
115126

116127
return sshAgent, sshAgent.Listen()

‎db/Migration.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,9 @@ func Migrate(d Store) error {
100100
}
101101

102102
didRun = true
103-
fmt.Printf("Executing migration %s (at %v)...\n", version.HumanoidVersion(), time.Now())
103+
fmt.Printf("Executing migration %s (at %v)...\n", version.HumanoidVersion(), time.Now().UTC())
104104
if err := d.ApplyMigration(version); err != nil {
105-
fmt.Printf("Rolling back %s (time: %v)...\n", version.HumanoidVersion(), time.Now())
105+
fmt.Printf("Rolling back %s (time: %v)...\n", version.HumanoidVersion(), time.Now().UTC())
106106
d.TryRollbackMigration(version)
107107
return err
108108
}

‎db/Repository.go

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package db
22

33
import (
44
"fmt"
5-
"os"
65
"path"
76
"regexp"
87
"strconv"
@@ -34,45 +33,22 @@ type Repository struct {
3433
}
3534

3635
func (r Repository) ClearCache() error {
37-
dir, err := os.Open(util.Config.TmpPath)
38-
if err != nil {
39-
return err
40-
}
41-
defer dir.Close()
42-
43-
files, err := dir.ReadDir(0)
44-
if err != nil {
45-
return err
46-
}
47-
48-
for _, f := range files {
49-
if !f.IsDir() {
50-
continue
51-
}
52-
if strings.HasPrefix(f.Name(), r.getDirNamePrefix()) {
53-
err = os.RemoveAll(path.Join(util.Config.TmpPath, f.Name()))
54-
if err != nil {
55-
return err
56-
}
57-
}
58-
}
59-
60-
return nil
36+
return util.ClearDir(util.Config.GetProjectTmpDir(r.ProjectID), true, r.getDirNamePrefix())
6137
}
6238

6339
func (r Repository) getDirNamePrefix() string {
6440
return "repository_" + strconv.Itoa(r.ID) + "_"
6541
}
6642

6743
func (r Repository) GetDirName(templateID int) string {
68-
return r.getDirNamePrefix() + strconv.Itoa(templateID)
44+
return r.getDirNamePrefix() + "template_" + strconv.Itoa(templateID)
6945
}
7046

7147
func (r Repository) GetFullPath(templateID int) string {
7248
if r.GetType() == RepositoryLocal {
7349
return r.GetGitURL(true)
7450
}
75-
return path.Join(util.Config.TmpPath, r.GetDirName(templateID))
51+
return path.Join(util.Config.GetProjectTmpDir(r.ProjectID), r.GetDirName(templateID))
7652
}
7753

7854
func (r Repository) GetGitURL(secure bool) string {

‎db/Repository_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func TestRepository_ClearCache(t *testing.T) {
2121
util.Config = &util.ConfigType{
2222
TmpPath: path.Join(os.TempDir(), util.RandString(rand.Intn(10-4)+4)),
2323
}
24-
repoDir := path.Join(util.Config.TmpPath, "repository_123_55")
24+
repoDir := path.Join(util.Config.TmpPath, "project_0", "repository_123_55")
2525
err := os.MkdirAll(repoDir, 0755)
2626
require.NoError(t, err)
2727

‎db/Runner.go

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,20 @@
11
package db
22

3-
type RunnerState string
3+
import "time"
44

5-
//const (
6-
// RunnerOffline RunnerState = "offline"
7-
// RunnerActive RunnerState = "active"
8-
//)
5+
type RunnerState string
96

107
type Runner struct {
11-
ID int `db:"id" json:"id"`
12-
Token string `db:"token" json:"-"`
13-
ProjectID *int `db:"project_id" json:"project_id"`
14-
//State RunnerState `db:"state" json:"state"`
15-
Webhook string `db:"webhook" json:"webhook"`
16-
MaxParallelTasks int `db:"max_parallel_tasks" json:"max_parallel_tasks"`
17-
Active bool `db:"active" json:"active"`
18-
Name string `db:"name" json:"name"`
19-
Tag string `db:"tag" json:"tag"`
8+
ID int `db:"id" json:"id"`
9+
Token string `db:"token" json:"-"`
10+
ProjectID *int `db:"project_id" json:"project_id"`
11+
Webhook string `db:"webhook" json:"webhook"`
12+
MaxParallelTasks int `db:"max_parallel_tasks" json:"max_parallel_tasks"`
13+
Active bool `db:"active" json:"active"`
14+
Name string `db:"name" json:"name"`
15+
Tag string `db:"tag" json:"tag"`
16+
Touched *time.Time `db:"touched" json:"touched"`
17+
CleaningRequested *time.Time `db:"cleaning_requested" json:"cleaning_requested"`
2018

2119
PublicKey *string `db:"public_key" json:"-"`
2220
}

‎db/Store.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ type Store interface {
357357
DeleteGlobalRunner(runnerID int) error
358358
UpdateRunner(runner Runner) error
359359
CreateRunner(runner Runner) (Runner, error)
360+
TouchRunner(runner Runner) (err error)
361+
ClearRunnerCache(runner Runner) (err error)
360362

361363
GetTemplateVaults(projectID int, templateID int) ([]TemplateVault, error)
362364
CreateTemplateVault(vault TemplateVault) (TemplateVault, error)

0 commit comments

Comments
 (0)