Skip to content

Commit 22153be

Browse files
pk910pk910
authored
Merge pull request #114 from ethpandaops/pk910/relative-includes
allow relative paths in `run_external_tasks`, resolving against parent base path/URL
2 parents e47284f + ee8a0f5 commit 22153be

File tree

9 files changed

+132
-53
lines changed

9 files changed

+132
-53
lines changed

‎pkg/coordinator/tasks/run_external_tasks/task.go‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"fmt"
66
"io"
77
"net/http"
8+
"net/url"
89
"os"
10+
"path"
911
"strings"
1012
"time"
1113

@@ -75,15 +77,22 @@ func (t *Task) LoadConfig() error {
7577
}
7678

7779
func (t *Task) Execute(ctx context.Context) error {
80+
// get task base path
81+
taskBasePath, ok := t.ctx.Vars.GetVar("taskBasePath").(string)
82+
if !ok {
83+
t.logger.Warn("could not read taskBasePath from variables")
84+
}
85+
7886
// load test yaml file
79-
testConfig, err := t.loadTestConfig(ctx, t.config.TestFile)
87+
testConfig, testBasePath, err := t.loadTestConfig(ctx, taskBasePath, t.config.TestFile)
8088
if err != nil {
8189
return err
8290
}
8391

8492
// create new variable scope for test configuration
8593
testVars := t.ctx.Vars.NewScope()
8694
testVars.SetVar("scopeOwner", uint64(t.ctx.Index))
95+
testVars.SetVar("taskBasePath", testBasePath)
8796
t.ctx.Outputs.SetSubScope("childScope", vars.NewScopeFilter(testVars))
8897

8998
// add default config from external test to variable scope
@@ -187,20 +196,34 @@ taskLoop:
187196
return resError
188197
}
189198

190-
func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.TestConfig, error) {
199+
func (t *Task) loadTestConfig(ctx context.Context, basePath, testFile string) (*types.TestConfig, string, error) {
191200
var reader io.Reader
192201

202+
var testBasePath string
203+
193204
if strings.HasPrefix(testFile, "http://") || strings.HasPrefix(testFile, "https://") {
205+
parsedURL, err := url.Parse(testFile)
206+
if err != nil {
207+
return nil, "", err
208+
}
209+
210+
// Remove the filename from the path
211+
parsedURL.Path = path.Dir(parsedURL.Path)
212+
parsedURL.RawQuery = ""
213+
parsedURL.Fragment = ""
214+
215+
testBasePath = parsedURL.String()
216+
194217
client := &http.Client{Timeout: time.Second * 120}
195218

196219
req, err := http.NewRequestWithContext(ctx, "GET", testFile, http.NoBody)
197220
if err != nil {
198-
return nil, err
221+
return nil, testBasePath, err
199222
}
200223

201224
resp, err := client.Do(req)
202225
if err != nil {
203-
return nil, err
226+
return nil, testBasePath, err
204227
}
205228

206229
defer func() {
@@ -210,14 +233,20 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
210233
}()
211234

212235
if resp.StatusCode != http.StatusOK {
213-
return nil, fmt.Errorf("error loading test config from url: %v, result: %v %v", testFile, resp.StatusCode, resp.Status)
236+
return nil, testBasePath, fmt.Errorf("error loading test config from url: %v, result: %v %v", testFile, resp.StatusCode, resp.Status)
214237
}
215238

216239
reader = resp.Body
217240
} else {
241+
if !path.IsAbs(testFile) && basePath != "" {
242+
testFile = path.Join(basePath, testFile)
243+
}
244+
245+
testBasePath = path.Dir(testFile)
246+
218247
f, err := os.Open(testFile)
219248
if err != nil {
220-
return nil, fmt.Errorf("error loading test config from file %v: %w", testFile, err)
249+
return nil, testBasePath, fmt.Errorf("error loading test config from file %v: %w", testFile, err)
221250
}
222251

223252
defer func() {
@@ -234,7 +263,7 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
234263

235264
err := decoder.Decode(testConfig)
236265
if err != nil {
237-
return nil, fmt.Errorf("error decoding external test config %v: %v", testFile, err)
266+
return nil, testBasePath, fmt.Errorf("error decoding external test config %v: %v", testFile, err)
238267
}
239268

240269
if testConfig.Config == nil {
@@ -245,5 +274,5 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
245274
testConfig.ConfigVars = map[string]string{}
246275
}
247276

248-
return testConfig, nil
277+
return testConfig, testBasePath, nil
249278
}

‎pkg/coordinator/test/descriptor.go‎

Lines changed: 60 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"fmt"
66
"io"
77
"net/http"
8+
"net/url"
89
"os"
10+
"path"
911
"strings"
1012
"time"
1113

@@ -15,25 +17,32 @@ import (
1517
)
1618

1719
type Descriptor struct {
18-
id string
19-
source string
20-
config *types.TestConfig
21-
vars types.Variables
22-
err error
20+
id string
21+
source string
22+
basePath string
23+
config *types.TestConfig
24+
vars types.Variables
25+
err error
2326
}
2427

25-
func NewDescriptor(testID, testSrc string, config *types.TestConfig, variables types.Variables) *Descriptor {
28+
func NewDescriptor(testID, testSrc, basePath string, config *types.TestConfig, variables types.Variables) *Descriptor {
2629
return &Descriptor{
27-
id: testID,
28-
source: testSrc,
29-
config: config,
30-
vars: variables,
30+
id: testID,
31+
source: testSrc,
32+
basePath: basePath,
33+
config: config,
34+
vars: variables,
3135
}
3236
}
3337

3438
func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localTests []*types.TestConfig, externalTests []*types.ExternalTestConfig) []types.TestDescriptor {
3539
descriptors := []types.TestDescriptor{}
3640

41+
workingDir, err := os.Getwd()
42+
if err != nil {
43+
logrus.WithError(err).Warn("failed to get working directory")
44+
}
45+
3746
// load local tests
3847
for testIdx, testCfg := range localTests {
3948
testID := testCfg.ID
@@ -52,11 +61,12 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
5261
err := testVars.CopyVars(globalVars, testCfg.ConfigVars)
5362

5463
descriptors = append(descriptors, &Descriptor{
55-
id: testID,
56-
source: testSrc,
57-
vars: testVars,
58-
config: localTests[testIdx],
59-
err: err,
64+
id: testID,
65+
source: testSrc,
66+
basePath: workingDir,
67+
vars: testVars,
68+
config: localTests[testIdx],
69+
err: err,
6070
})
6171
}
6272

@@ -65,7 +75,7 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
6575
testSrc := fmt.Sprintf("external:%v", extTestCfg.File)
6676
testID := ""
6777

68-
testConfig, testVars, err := LoadExternalTestConfig(ctx, globalVars, extTestCfg)
78+
testConfig, testVars, basePath, err := LoadExternalTestConfig(ctx, globalVars, extTestCfg)
6979

7080
if testConfig != nil && testConfig.ID != "" {
7181
testID = testConfig.ID
@@ -76,31 +86,46 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
7686
}
7787

7888
descriptors = append(descriptors, &Descriptor{
79-
id: testID,
80-
source: testSrc,
81-
config: testConfig,
82-
vars: testVars,
83-
err: err,
89+
id: testID,
90+
source: testSrc,
91+
basePath: basePath,
92+
config: testConfig,
93+
vars: testVars,
94+
err: err,
8495
})
8596
}
8697

8798
return descriptors
8899
}
89100

90-
func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, extTestCfg *types.ExternalTestConfig) (*types.TestConfig, types.Variables, error) {
101+
func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, extTestCfg *types.ExternalTestConfig) (*types.TestConfig, types.Variables, string, error) {
91102
var reader io.Reader
92103

104+
var basePath string
105+
93106
if strings.HasPrefix(extTestCfg.File, "http://") || strings.HasPrefix(extTestCfg.File, "https://") {
107+
parsedURL, err := url.Parse(extTestCfg.File)
108+
if err != nil {
109+
return nil, nil, "", err
110+
}
111+
112+
// Remove the filename from the path
113+
parsedURL.Path = path.Dir(parsedURL.Path)
114+
parsedURL.RawQuery = ""
115+
parsedURL.Fragment = ""
116+
117+
basePath = parsedURL.String()
118+
94119
client := &http.Client{Timeout: time.Second * 120}
95120

96121
req, err := http.NewRequestWithContext(ctx, "GET", extTestCfg.File, http.NoBody)
97122
if err != nil {
98-
return nil, nil, err
123+
return nil, nil, basePath, err
99124
}
100125

101126
resp, err := client.Do(req)
102127
if err != nil {
103-
return nil, nil, err
128+
return nil, nil, basePath, err
104129
}
105130

106131
defer func() {
@@ -110,14 +135,16 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
110135
}()
111136

112137
if resp.StatusCode != http.StatusOK {
113-
return nil, nil, fmt.Errorf("error loading test config from url: %v, result: %v %v", extTestCfg.File, resp.StatusCode, resp.Status)
138+
return nil, nil, basePath, fmt.Errorf("error loading test config from url: %v, result: %v %v", extTestCfg.File, resp.StatusCode, resp.Status)
114139
}
115140

116141
reader = resp.Body
117142
} else {
143+
basePath = path.Dir(extTestCfg.File)
144+
118145
f, err := os.Open(extTestCfg.File)
119146
if err != nil {
120-
return nil, nil, fmt.Errorf("error loading test config from file %v: %w", extTestCfg.File, err)
147+
return nil, nil, basePath, fmt.Errorf("error loading test config from file %v: %w", extTestCfg.File, err)
121148
}
122149

123150
defer func() {
@@ -135,7 +162,7 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
135162

136163
err := decoder.Decode(testConfig)
137164
if err != nil {
138-
return nil, nil, fmt.Errorf("error decoding external test config %v: %v", extTestCfg.File, err)
165+
return nil, nil, basePath, fmt.Errorf("error decoding external test config %v: %v", extTestCfg.File, err)
139166
}
140167

141168
if testConfig.Config == nil {
@@ -177,10 +204,10 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
177204

178205
err = testVars.CopyVars(testVars, testConfig.ConfigVars)
179206
if err != nil {
180-
return nil, nil, fmt.Errorf("error decoding external test configVars %v: %v", extTestCfg.File, err)
207+
return nil, nil, basePath, fmt.Errorf("error decoding external test configVars %v: %v", extTestCfg.File, err)
181208
}
182209

183-
return testConfig, testVars, nil
210+
return testConfig, testVars, basePath, nil
184211
}
185212

186213
func (d *Descriptor) ID() string {
@@ -191,6 +218,10 @@ func (d *Descriptor) Source() string {
191218
return d.source
192219
}
193220

221+
func (d *Descriptor) BasePath() string {
222+
return d.basePath
223+
}
224+
194225
func (d *Descriptor) Config() *types.TestConfig {
195226
return d.config
196227
}

‎pkg/coordinator/test/test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ func CreateTest(runID uint64, descriptor types.TestDescriptor, log logrus.FieldL
5252
test.variables.SetVar(cfgKey, cfgValue)
5353
}
5454

55+
// set base path
56+
test.variables.SetVar("testBasePath", descriptor.BasePath())
57+
test.variables.SetVar("taskBasePath", descriptor.BasePath())
58+
5559
// add test run to database
5660
configYaml, err := yaml.Marshal(test.variables.GetVarsMap(nil, false))
5761
if err != nil {

‎pkg/coordinator/testregistry.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"os"
78
"sort"
89
"sync"
910
"time"
@@ -205,7 +206,12 @@ func (c *TestRegistry) AddLocalTest(testConfig *types.TestConfig) (types.TestDes
205206
return nil, fmt.Errorf("failed decoding configVars: %v", err)
206207
}
207208

208-
testDescriptor := test.NewDescriptor(testConfig.ID, "api-call", testConfig, testVars)
209+
workingDir, err := os.Getwd()
210+
if err != nil {
211+
return nil, fmt.Errorf("failed getting working directory: %v", err)
212+
}
213+
214+
testDescriptor := test.NewDescriptor(testConfig.ID, "api-call", workingDir, testConfig, testVars)
209215

210216
c.testDescriptorsMutex.Lock()
211217
defer c.testDescriptorsMutex.Unlock()
@@ -225,7 +231,7 @@ func (c *TestRegistry) AddLocalTest(testConfig *types.TestConfig) (types.TestDes
225231
}
226232

227233
func (c *TestRegistry) AddExternalTest(ctx context.Context, extTestCfg *types.ExternalTestConfig) (types.TestDescriptor, error) {
228-
testConfig, testVars, err := test.LoadExternalTestConfig(ctx, c.coordinator.GlobalVariables(), extTestCfg)
234+
testConfig, testVars, basePath, err := test.LoadExternalTestConfig(ctx, c.coordinator.GlobalVariables(), extTestCfg)
229235
if err != nil {
230236
return nil, fmt.Errorf("failed loading test config from %v: %w", extTestCfg.File, err)
231237
}
@@ -242,7 +248,7 @@ func (c *TestRegistry) AddExternalTest(ctx context.Context, extTestCfg *types.Ex
242248
return nil, errors.New("test must have 1 or more tasks")
243249
}
244250

245-
testDescriptor := test.NewDescriptor(testConfig.ID, fmt.Sprintf("external:%v", extTestCfg.File), testConfig, testVars)
251+
testDescriptor := test.NewDescriptor(testConfig.ID, fmt.Sprintf("external:%v", extTestCfg.File), basePath, testConfig, testVars)
246252
extTestCfg.ID = testDescriptor.ID()
247253
extTestCfg.Name = testConfig.Name
248254

‎pkg/coordinator/types/test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ type TestSchedule struct {
6969
type TestDescriptor interface {
7070
ID() string
7171
Source() string
72+
BasePath() string
7273
Config() *TestConfig
7374
Vars() Variables
7475
Err() error

‎pkg/coordinator/web/api/get_test_api.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
type GetTestResponse struct {
1111
ID string `json:"id"`
1212
Source string `json:"source"`
13+
BasePath string `json:"basePath"`
1314
Name string `json:"name"`
1415
Timeout uint64 `json:"timeout"`
1516
Config map[string]any `json:"config"`
@@ -60,6 +61,7 @@ func (ah *APIHandler) GetTest(w http.ResponseWriter, r *http.Request) {
6061
ah.sendOKResponse(w, r.URL.String(), &GetTestResponse{
6162
ID: testDescriptor.ID(),
6263
Source: testDescriptor.Source(),
64+
BasePath: testDescriptor.BasePath(),
6365
Name: testConfig.Name,
6466
Timeout: uint64(testConfig.Timeout.Seconds()),
6567
Config: testConfig.Config,

0 commit comments

Comments
 (0)