Skip to content

Commit 69e911f

Browse files
jamsman94jamsman94
authored
upload file in workflow feature (#4307)
* upload file in workflow feature Signed-off-by: Min Min <jamsman94@gmail.com> * add missing files Signed-off-by: Min Min <jamsman94@gmail.com> * debug logs Signed-off-by: Min Min <jamsman94@gmail.com> * debug logs Signed-off-by: Min Min <jamsman94@gmail.com> * debug Signed-off-by: Min Min <jamsman94@gmail.com> * fix missing file info Signed-off-by: Min Min <jamsman94@gmail.com> * change logic Signed-off-by: Min Min <jamsman94@gmail.com> * fix missing fields Signed-off-by: Min Min <jamsman94@gmail.com> * add download API Signed-off-by: Min Min <jamsman94@gmail.com> * fix download panic problem Signed-off-by: Min Min <jamsman94@gmail.com> * change agent logic Signed-off-by: Min Min <jamsman94@gmail.com> * debug Signed-off-by: Min Min <jamsman94@gmail.com> * update agent logic Signed-off-by: Min Min <jamsman94@gmail.com> * change gloo authentication code Signed-off-by: Min Min <jamsman94@gmail.com> * improve logic for vm jobs Signed-off-by: Min Min <jamsman94@gmail.com> * change some basic data structure and update download file logic for agent Signed-off-by: Min Min <jamsman94@gmail.com> * fix directory not found error Signed-off-by: Min Min <jamsman94@gmail.com> * debug Signed-off-by: Min Min <jamsman94@gmail.com> * enhance cleanup logic and add more debug logs Signed-off-by: Min Min <jamsman94@gmail.com> * compatibility for large files Signed-off-by: Min Min <jamsman94@gmail.com> * more debug logs Signed-off-by: Min Min <jamsman94@gmail.com> * possible fix to the hanging problem Signed-off-by: Min Min <jamsman94@gmail.com> * multiple changes to the file uploading logic Signed-off-by: Min Min <jamsman94@gmail.com> * debug Signed-off-by: Min Min <jamsman94@gmail.com> * format code Signed-off-by: Min Min <jamsman94@gmail.com> * change mounting logic and add root handling Signed-off-by: Min Min <jamsman94@gmail.com> * slightly improve logic Signed-off-by: Min Min <jamsman94@gmail.com> * remove overdesign and change logging Signed-off-by: Min Min <jamsman94@gmail.com> * update mounting to root logic so the pod won't crash Signed-off-by: Min Min <jamsman94@gmail.com> * fix volume name error Signed-off-by: Min Min <jamsman94@gmail.com> * fix directory problem Signed-off-by: Min Min <jamsman94@gmail.com> * change fileName logic Signed-off-by: Min Min <jamsman94@gmail.com> * fix mount multiple file in one directory with different depth Signed-off-by: Min Min <jamsman94@gmail.com> * fix error message Signed-off-by: Min Min <jamsman94@gmail.com> --------- Signed-off-by: Min Min <jamsman94@gmail.com>
1 parent c5e64ee commit 69e911f

25 files changed

Lines changed: 2466 additions & 51 deletions

File tree

‎pkg/cli/zadig-agent/internal/agent/job/job_executor.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@ func (e *JobExecutor) BeforeExecute() error {
8888
return err
8989
}
9090

91+
// Download files if any are specified
92+
err = e.downloadJobFiles()
93+
if err != nil {
94+
log.Errorf("failed to download job files, error: %v", err)
95+
return err
96+
}
97+
9198
return nil
9299
}
93100

@@ -428,3 +435,75 @@ func (e *JobExecutor) CheckZadigCancel() bool {
428435
}
429436
return false
430437
}
438+
439+
// downloadJobFiles downloads all files specified in JobCtx.Files to the workspace
440+
func (e *JobExecutor) downloadJobFiles() error {
441+
if len(e.JobCtx.Files) == 0 {
442+
return nil // No files to download
443+
}
444+
445+
log.Infof("Starting to download %d file(s) for job %s", len(e.JobCtx.Files), e.Job.JobName)
446+
447+
// Download each file using the directory info from fileInfo
448+
for _, fileInfo := range e.JobCtx.Files {
449+
if err := e.downloadSingleFile(fileInfo); err != nil {
450+
return fmt.Errorf("failed to download file %s (ID: %s): %v", fileInfo.FileName, fileInfo.FileID, err)
451+
}
452+
}
453+
454+
e.Logger.Infof("Successfully downloaded all %d file(s) for job %s", len(e.JobCtx.Files), e.Job.JobName)
455+
return nil
456+
}
457+
458+
// downloadSingleFile downloads a single file and updates the environment variable
459+
func (e *JobExecutor) downloadSingleFile(fileInfo *jobctl.JobFileInfo) error {
460+
// Compute filename as the last segment of the provided path, and targetDir as the path without that segment.
461+
// Validate and normalize similarly to controller logic, preventing workspace escapes for relative inputs.
462+
mp := strings.TrimSpace(fileInfo.FilePath)
463+
if mp == "" {
464+
return fmt.Errorf("file env %s has empty path", fileInfo.EnvKey)
465+
}
466+
467+
cleanMP := filepath.Clean(mp)
468+
fileName := filepath.Base(cleanMP)
469+
dirComponent := filepath.Dir(cleanMP)
470+
471+
// Handle degenerate cases where Base returns "." or root
472+
if fileName == "." || fileName == string(os.PathSeparator) || fileName == "" {
473+
// Fallback to provided names
474+
if fileInfo.FileName != "" {
475+
fileName = fileInfo.FileName
476+
} else {
477+
fileName = fileInfo.EnvKey
478+
}
479+
// In this case, treat the entire cleanMP as the directory component
480+
dirComponent = cleanMP
481+
}
482+
483+
var targetDir string
484+
if filepath.IsAbs(cleanMP) {
485+
targetDir = filepath.Clean(dirComponent)
486+
} else {
487+
targetDir = filepath.Clean(filepath.Join(e.Dirs.Workspace, dirComponent))
488+
// Ensure the cleaned path is still within the workspace
489+
ws := filepath.Clean(e.Dirs.Workspace)
490+
wsPrefix := ws
491+
if !strings.HasSuffix(wsPrefix, string(os.PathSeparator)) {
492+
wsPrefix = wsPrefix + string(os.PathSeparator)
493+
}
494+
if targetDir != ws && !strings.HasPrefix(targetDir, wsPrefix) {
495+
return fmt.Errorf("relative path for %s escapes workspace: %s", fileInfo.EnvKey, targetDir)
496+
}
497+
}
498+
499+
targetPath := filepath.Join(targetDir, fileName)
500+
501+
log.Infof("Downloading file %s (ID: %s) to %s", fileInfo.FileName, fileInfo.FileID, targetPath)
502+
503+
if err := e.Client.DownloadFile(fileInfo.FileID, fileName, targetDir); err != nil {
504+
return fmt.Errorf("failed to download file %s: %v", fileInfo.FileName, err)
505+
}
506+
507+
e.Logger.Infof("Successfully downloaded %s and set environment variable %s=%s", fileInfo.FileName, fileInfo.EnvKey, targetPath)
508+
return nil
509+
}

‎pkg/cli/zadig-agent/internal/network/connect.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ import (
2424
)
2525

2626
const (
27-
RegisterBaseUrl = "/api/aslan/vm/agents/register"
28-
VerifyBaseUrl = "/api/aslan/vm/agents/verify"
29-
heartbeatBaseUrl = "/api/aslan/vm/agents/heartbeat"
30-
RequestJobBaseUrl = "/api/aslan/vm/agents/job/request"
31-
ReportJobBaseUrl = "/api/aslan/vm/agents/job/report"
27+
RegisterBaseUrl = "/api/aslan/vm/agents/register"
28+
VerifyBaseUrl = "/api/aslan/vm/agents/verify"
29+
heartbeatBaseUrl = "/api/aslan/vm/agents/heartbeat"
30+
RequestJobBaseUrl = "/api/aslan/vm/agents/job/request"
31+
ReportJobBaseUrl = "/api/aslan/vm/agents/job/report"
32+
DownloadFileBaseUrl = "/api/aslan/vm/agents/tempFile/download/%s"
3233
)
3334

3435
type RegisterAgentParameters struct {

‎pkg/cli/zadig-agent/internal/network/network.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ package network
1818

1919
import (
2020
"fmt"
21+
"os"
22+
"path/filepath"
2123

2224
"github.com/koderover/zadig/v2/pkg/cli/zadig-agent/config"
2325
"github.com/koderover/zadig/v2/pkg/cli/zadig-agent/internal/common/types"
@@ -105,3 +107,25 @@ func (c *ZadigClient) ReportJob(parameters *types.ReportJobParameters) (*types.R
105107
}
106108
return nil, fmt.Errorf("failed to report job to zadig server")
107109
}
110+
111+
type DownloadFileRequest struct {
112+
Token string `json:"token"`
113+
}
114+
115+
// DownloadFile downloads a file from the server using the file ID
116+
func (c *ZadigClient) DownloadFile(fileID, fileName, targetDir string) error {
117+
downloadURL := GetFullURL(c.AgentConfig.URL, fmt.Sprintf(DownloadFileBaseUrl+"?token=%s", fileID, c.AgentConfig.Token))
118+
119+
// Ensure the target directory exists
120+
if err := os.MkdirAll(targetDir, 0755); err != nil {
121+
return fmt.Errorf("failed to create target directory %s: %v", targetDir, err)
122+
}
123+
124+
targetPath := filepath.Join(targetDir, fileName)
125+
err := httpclient.Download(downloadURL, targetPath)
126+
if err != nil {
127+
return fmt.Errorf("failed to download file (ID: %s) to %s: %v", fileID, targetPath, err)
128+
}
129+
130+
return nil
131+
}

‎pkg/microservice/aslan/core/common/repository/models/build.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,20 @@ package models
1919
import (
2020
"strings"
2121

22-
"github.com/koderover/zadig/v2/pkg/microservice/aslan/config"
23-
"github.com/koderover/zadig/v2/pkg/util"
2422
"go.mongodb.org/mongo-driver/bson/primitive"
2523

24+
"github.com/koderover/zadig/v2/pkg/microservice/aslan/config"
2625
"github.com/koderover/zadig/v2/pkg/setting"
2726
"github.com/koderover/zadig/v2/pkg/types"
27+
"github.com/koderover/zadig/v2/pkg/util"
2828
)
2929

30+
// FileNameResolver is a function type to resolve file ID to file name
31+
type FileNameResolver func(fileID string) (string, error)
32+
33+
// Global file name resolver function - will be set by the service layer
34+
var GetFileNameByID FileNameResolver
35+
3036
type Build struct {
3137
ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"`
3238
Name string `bson:"name" json:"name"`
@@ -221,6 +227,8 @@ type KeyVal struct {
221227
Script string `bson:"script,omitempty" json:"script,omitempty" yaml:"script,omitempty"`
222228
CallFunction string `bson:"call_function,omitempty" json:"call_function,omitempty" yaml:"call_function,omitempty"`
223229
FunctionReference []string `bson:"function_reference,omitempty" json:"function_reference,omitempty" yaml:"function_reference,omitempty"`
230+
FilePath string `bson:"file_path,omitempty" json:"file_path,omitempty" yaml:"file_path,omitempty"`
231+
FileID string `bson:"file_id,omitempty" json:"file_id,omitempty" yaml:"file_id,omitempty"`
224232
IsCredential bool `bson:"is_credential" json:"is_credential" yaml:"is_credential"`
225233
Description string `bson:"description" json:"description" yaml:"description"`
226234
}
@@ -229,6 +237,9 @@ func (kv *KeyVal) GetValue() string {
229237
if kv.Type == MultiSelectType {
230238
return strings.Join(kv.ChoiceValue, ",")
231239
}
240+
if kv.Type == FileType {
241+
return kv.FilePath
242+
}
232243
return kv.Value
233244
}
234245

‎pkg/microservice/aslan/core/common/repository/models/temporary_file.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
Copyright 2025 The KodeRover Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package models
18+
19+
import (
20+
"go.mongodb.org/mongo-driver/bson/primitive"
21+
)
22+
23+
const (
24+
TemporaryFileStatusUploading = "uploading"
25+
TemporaryFileStatusCompleted = "completed"
26+
TemporaryFileStatusFailed = "failed"
27+
TemporaryFileStatusExpired = "expired"
28+
)
29+
30+
type TemporaryFile struct {
31+
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
32+
SessionID string `bson:"session_id" json:"session_id"`
33+
FileName string `bson:"file_name" json:"file_name"`
34+
FileSize int64 `bson:"file_size" json:"file_size"`
35+
FileHash string `bson:"file_hash" json:"file_hash"`
36+
TotalParts int `bson:"total_parts" json:"total_parts"`
37+
UploadedParts []int `bson:"uploaded_parts" json:"uploaded_parts"`
38+
Status string `bson:"status" json:"status"`
39+
FilePath string `bson:"file_path" json:"file_path"`
40+
StorageID string `bson:"storage_id" json:"storage_id"`
41+
InstanceID string `bson:"instance_id" json:"instance_id"`
42+
CreatedAt int64 `bson:"created_at" json:"created_at"`
43+
UpdatedAt int64 `bson:"updated_at" json:"updated_at"`
44+
ExpiresAt int64 `bson:"expires_at" json:"expires_at"`
45+
}
46+
47+
func (TemporaryFile) TableName() string {
48+
return "temporary_file"
49+
}
50+
51+
type InitiateUploadRequest struct {
52+
FileName string `json:"file_name" binding:"required"`
53+
FileSize int64 `json:"file_size" binding:"required"`
54+
TotalParts int `json:"total_parts" binding:"required"`
55+
}
56+
57+
type InitiateUploadResponse struct {
58+
SessionID string `json:"session_id"`
59+
}
60+
61+
type UploadStatusResponse struct {
62+
SessionID string `json:"session_id"`
63+
Status string `json:"status"`
64+
FileName string `json:"file_name"`
65+
FileSize int64 `json:"file_size"`
66+
TotalParts int `json:"total_parts"`
67+
UploadedParts []int `json:"uploaded_parts"`
68+
Progress float64 `json:"progress"`
69+
CreatedAt int64 `json:"created_at"`
70+
UpdatedAt int64 `json:"updated_at"`
71+
}
72+
73+
type CompleteUploadRequest struct {
74+
FileHash string `json:"file_hash" binding:"required"`
75+
}
76+
77+
type CompleteUploadResponse struct {
78+
FileID string `json:"file_id"`
79+
FilePath string `json:"file_path"`
80+
}

‎pkg/microservice/aslan/core/common/repository/models/workflow_v4.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ const (
120120
MultiSelectType ParameterSettingType = "multi-select"
121121
ImageType ParameterSettingType = "image"
122122
Script ParameterSettingType = "script"
123+
FileType ParameterSettingType = "file"
123124
// Deprecated
124125
ExternalType ParameterSettingType = "external"
125126
)
@@ -1543,12 +1544,14 @@ type GeneralHook struct {
15431544
type Param struct {
15441545
Name string `bson:"name" json:"name" yaml:"name"`
15451546
Description string `bson:"description" json:"description" yaml:"description"`
1546-
// support string/text/choice/repo type
1547+
// support string/text/choice/repo/file type
15471548
ParamsType string `bson:"type" json:"type" yaml:"type"`
15481549
Value string `bson:"value" json:"value" yaml:"value,omitempty"`
15491550
Repo *types.Repository `bson:"repo" json:"repo" yaml:"repo,omitempty"`
15501551
ChoiceOption []string `bson:"choice_option,omitempty" json:"choice_option,omitempty" yaml:"choice_option,omitempty"`
15511552
ChoiceValue []string `bson:"choice_value,omitempty" json:"choice_value,omitempty" yaml:"choice_value,omitempty"`
1553+
FileID string `bson:"file_id,omitempty" json:"file_id,omitempty" yaml:"file_id,omitempty"`
1554+
FilePath string `bson:"file_path,omitempty" json:"file_path,omitempty" yaml:"file_path,omitempty"`
15521555
Default string `bson:"default" json:"default" yaml:"default"`
15531556
IsCredential bool `bson:"is_credential" json:"is_credential" yaml:"is_credential"`
15541557
Source config.ParamSourceType `bson:"source,omitempty" json:"source,omitempty" yaml:"source,omitempty"`
@@ -1558,9 +1561,30 @@ func (p *Param) GetValue() string {
15581561
if p.ParamsType == "multi-select" {
15591562
return strings.Join(p.ChoiceValue, ",")
15601563
}
1564+
if p.ParamsType == "file" {
1565+
return p.GetFileValue()
1566+
}
15611567
return p.Value
15621568
}
15631569

1570+
// GetFileValue returns the file path with /zadig_files/ prefix using the actual fileName
1571+
func (p *Param) GetFileValue() string {
1572+
if p.FileID == "" {
1573+
return ""
1574+
}
1575+
1576+
// Use the global resolver function if available
1577+
if GetFileNameByID != nil {
1578+
fileName, err := GetFileNameByID(p.FileID)
1579+
if err == nil && fileName != "" {
1580+
return fmt.Sprintf("%s/%s", p.FilePath, fileName)
1581+
}
1582+
}
1583+
1584+
// Fallback: return empty string if we can't resolve the file name
1585+
return ""
1586+
}
1587+
15641588
type ShareStorage struct {
15651589
Name string `bson:"name" json:"name" yaml:"name"`
15661590
Path string `bson:"path" json:"path" yaml:"path"`

0 commit comments

Comments
 (0)