Skip to content

Commit 6a3d08f

Browse files
samueltaripinyockgengithub-actions[bot]
authored
Local package repo support (#510)
* Adding local package repositories interface Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> * Adding create temp local repo interfaces Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> * Adding rpm repo creation and web repo functions Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> * Enabling local repo creation tool for rpm Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> * Adding debian local repo creation functions Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> * fix local repo * fix schema * remove not required host dependency * increase unit test coverage * increase unit test coverage * create unit test for schema * increase unit test for weak path * increase unit test coverage * fix local repo for rpm * address copilot comment * fix failing test * remove failing test * chore: auto-update coverage threshold to 66.2% (was 65.9%) * address comment * fix lint --------- Signed-off-by: Mah, Yock Gen <yock.gen.mah@intel.com> Co-authored-by: Mah, Yock Gen <yock.gen.mah@intel.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent 455dc8e commit 6a3d08f

27 files changed

+3230
-36
lines changed

‎.coverage-threshold‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
65.9
1+
66.2

‎image-templates/emt3-x86_64-edge-raw.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ packageRepositories:
4141
pkey: "<PUBLIC_KEY_URL>" # Uncomment and replace in real config
4242
component: "restricted"
4343

44+
# - codename: "localrpm"
45+
# path: "/data/os-image-composer/localrpm" # Uncomment and replace with local path
46+
# pkey: "[trusted=yes]" # Make sure to put trusted
47+
# component: "main" # Make sure to put main or leave empty
48+
4449
# Disk configuration can be omitted to use defaults from default template
4550
# If specified, it will override the default disk configuration completely
4651

‎image-templates/ubuntu24-x86_64-edge-raw.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ packageRepositories:
4040
pkey: "<PUBLIC_KEY_URL>" # Uncomment and replace in real config
4141
component: "restricted"
4242

43+
# - codename: "localdeb"
44+
# path: "/data/os-image-composer/localdeb" # Uncomment and replace with local path
45+
# pkey: "[trusted=yes]" # Make sure to put trusted
46+
# component: "main" # Make sure to put main or leave empty
47+
4348
systemConfig:
4449
name: edge
4550
description: edge ubuntu image with immutable rootfs

‎internal/config/apt_sources_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,34 @@ func TestGenerateAptSourcesFromRepositories_NonDEBSystem(t *testing.T) {
387387
}
388388
}
389389

390+
func TestGenerateAptSourcesFromRepositories_PathOnlyRepo(t *testing.T) {
391+
template := &ImageTemplate{
392+
Target: TargetInfo{
393+
OS: "ubuntu",
394+
},
395+
PackageRepositories: []PackageRepository{
396+
{
397+
Codename: "localdeb",
398+
Path: "/data/os-image-composer/localdeb",
399+
PKey: "[trusted=yes]",
400+
Component: "main",
401+
},
402+
},
403+
SystemConfig: SystemConfig{
404+
AdditionalFiles: []AdditionalFileInfo{},
405+
},
406+
}
407+
408+
err := template.GenerateAptSourcesFromRepositories()
409+
if err != nil {
410+
t.Fatalf("GenerateAptSourcesFromRepositories() failed for path-only repo: %v", err)
411+
}
412+
413+
if len(template.SystemConfig.AdditionalFiles) != 0 {
414+
t.Errorf("expected no apt additional files for path-only repo, got %d", len(template.SystemConfig.AdditionalFiles))
415+
}
416+
}
417+
390418
func TestAddUniqueAdditionalFile(t *testing.T) {
391419
template := &ImageTemplate{
392420
SystemConfig: SystemConfig{
@@ -618,6 +646,13 @@ func TestNormalizeRepositoryPriorities(t *testing.T) {
618646
}
619647
}
620648

649+
func TestNormalizeRepositoryPriorities_NilInput(t *testing.T) {
650+
result := normalizeRepositoryPriorities(nil)
651+
if len(result) != 0 {
652+
t.Errorf("expected empty result for nil input, got len=%d", len(result))
653+
}
654+
}
655+
621656
func TestGetRepositoryName(t *testing.T) {
622657
tests := []struct {
623658
name string

‎internal/config/config.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ type DiskConfig struct {
4545
type PackageRepository struct {
4646
ID string `yaml:"id,omitempty"` // Auto-assigned
4747
Codename string `yaml:"codename"` // Repository identifier/codename
48-
URL string `yaml:"url"` // Repository base URL
48+
URL string `yaml:"url,omitempty"` // Repository base URL
49+
Path string `yaml:"path,omitempty"` // Local directory path for file-based repositories
4950
PKey string `yaml:"pkey"` // Public GPG key URL for verification
5051
PKeys []string `yaml:"pkeys,omitempty"` // Multiple public GPG key URLs for verification
5152
Component string `yaml:"component,omitempty"` // Repository component (e.g., "main", "restricted")
@@ -268,6 +269,10 @@ func parseYAMLTemplate(data []byte, validateFull bool) (*ImageTemplate, error) {
268269
return nil, fmt.Errorf("template parsing failed: invalid structure: %w", err)
269270
}
270271

272+
if err := template.validatePackageRepositories(); err != nil {
273+
return nil, err
274+
}
275+
271276
return &template, nil
272277
}
273278

@@ -946,3 +951,24 @@ func (i *ImmutabilityConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
946951
func (i *ImmutabilityConfig) WasProvided() bool {
947952
return i.wasProvided
948953
}
954+
955+
func (t *ImageTemplate) validatePackageRepositories() error {
956+
for _, repo := range t.PackageRepositories {
957+
if err := repo.ValidatePackageRepository(); err != nil {
958+
return err
959+
}
960+
}
961+
962+
return nil
963+
}
964+
965+
// ValidatePackageRepository validates that either URL or Path is provided
966+
func (pr *PackageRepository) ValidatePackageRepository() error {
967+
if pr.URL == "" && pr.Path == "" {
968+
return fmt.Errorf("repository '%s': either 'url' or 'path' must be provided", pr.Codename)
969+
}
970+
if pr.URL != "" && pr.Path != "" {
971+
return fmt.Errorf("repository '%s': cannot specify both 'url' and 'path', choose one", pr.Codename)
972+
}
973+
return nil
974+
}

‎internal/config/config_test.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2427,6 +2427,55 @@ systemConfig:
24272427
}
24282428
}
24292429

2430+
func TestLoadTemplateRejectsInvalidPackageRepository(t *testing.T) {
2431+
yamlContent := `image:
2432+
name: test-invalid-repo
2433+
version: "1.0.0"
2434+
2435+
target:
2436+
os: azure-linux
2437+
dist: azl3
2438+
arch: x86_64
2439+
imageType: raw
2440+
2441+
packageRepositories:
2442+
- codename: "invalid-repo"
2443+
url: "https://example.com/repo"
2444+
path: "/tmp/repo"
2445+
pkey: "https://example.com/key.pub"
2446+
2447+
systemConfig:
2448+
name: test
2449+
packages:
2450+
- test-package
2451+
kernel:
2452+
version: "6.12"
2453+
cmdline: "quiet"
2454+
`
2455+
2456+
tmpFile, err := os.CreateTemp("", "test-invalid-repo-*.yml")
2457+
if err != nil {
2458+
t.Fatalf("failed to create temp file: %v", err)
2459+
}
2460+
if err := tmpFile.Chmod(0600); err != nil {
2461+
tmpFile.Close()
2462+
os.Remove(tmpFile.Name())
2463+
t.Fatalf("failed to set temp file permissions: %v", err)
2464+
}
2465+
defer os.Remove(tmpFile.Name())
2466+
2467+
if _, err := tmpFile.WriteString(yamlContent); err != nil {
2468+
tmpFile.Close()
2469+
t.Fatalf("failed to write temp file: %v", err)
2470+
}
2471+
tmpFile.Close()
2472+
2473+
_, err = LoadTemplate(tmpFile.Name(), false)
2474+
if err == nil {
2475+
t.Fatal("expected LoadTemplate to reject invalid package repository configuration")
2476+
}
2477+
}
2478+
24302479
func TestGlobalConfigSaveWithCreateDirectory(t *testing.T) {
24312480
config := &GlobalConfig{
24322481
Workers: 4,
@@ -3206,6 +3255,77 @@ systemConfig:
32063255
}
32073256
}
32083257

3258+
func TestPackageRepositoryYAMLParsingLocalPath(t *testing.T) {
3259+
yamlContent := `image:
3260+
name: test-local-repo-parsing
3261+
version: "1.0.0"
3262+
3263+
target:
3264+
os: ubuntu
3265+
dist: ubuntu24
3266+
arch: x86_64
3267+
imageType: raw
3268+
3269+
packageRepositories:
3270+
- codename: "localdeb"
3271+
path: "/data/os-image-composer/localdeb"
3272+
pkey: "[trusted=yes]"
3273+
component: "main"
3274+
3275+
systemConfig:
3276+
name: test
3277+
packages:
3278+
- test-package
3279+
kernel:
3280+
version: "6.12"
3281+
cmdline: "quiet"
3282+
`
3283+
3284+
tmpFile, err := os.CreateTemp("", "test-local-repo-*.yml")
3285+
if err != nil {
3286+
t.Fatalf("failed to create temp file: %v", err)
3287+
}
3288+
if err := tmpFile.Chmod(0600); err != nil {
3289+
tmpFile.Close()
3290+
os.Remove(tmpFile.Name())
3291+
return
3292+
}
3293+
defer os.Remove(tmpFile.Name())
3294+
3295+
if _, err := tmpFile.WriteString(yamlContent); err != nil {
3296+
t.Fatalf("failed to write temp file: %v", err)
3297+
}
3298+
tmpFile.Close()
3299+
3300+
template, err := LoadTemplate(tmpFile.Name(), false)
3301+
if err != nil {
3302+
t.Fatalf("failed to load YAML template with local package repository: %v", err)
3303+
}
3304+
3305+
repos := template.GetPackageRepositories()
3306+
if len(repos) != 1 {
3307+
t.Fatalf("expected 1 parsed repository, got %d", len(repos))
3308+
}
3309+
3310+
repo := template.GetRepositoryByCodename("localdeb")
3311+
if repo == nil {
3312+
t.Fatalf("expected to find localdeb repository")
3313+
}
3314+
3315+
if repo.Path != "/data/os-image-composer/localdeb" {
3316+
t.Errorf("expected repo path '/data/os-image-composer/localdeb', got '%s'", repo.Path)
3317+
}
3318+
if repo.PKey != "[trusted=yes]" {
3319+
t.Errorf("expected repo pkey '[trusted=yes]', got '%s'", repo.PKey)
3320+
}
3321+
if repo.Component != "main" {
3322+
t.Errorf("expected repo component 'main', got '%s'", repo.Component)
3323+
}
3324+
if repo.URL != "" {
3325+
t.Errorf("expected repo URL to be empty for local path repository, got '%s'", repo.URL)
3326+
}
3327+
}
3328+
32093329
func TestPackageRepositoriesWithDuplicateCodenames(t *testing.T) {
32103330
repos := []PackageRepository{
32113331
{Codename: "duplicate", URL: "https://first.com", PKey: "https://first.com/key.pub"},

‎internal/config/schema/embed_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package schema
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
)
7+
8+
func TestEmbeddedSchemasNonEmpty(t *testing.T) {
9+
tests := []struct {
10+
name string
11+
schema []byte
12+
}{
13+
{"ImageTemplateSchema", ImageTemplateSchema},
14+
{"ConfigSchema", ConfigSchema},
15+
{"ChrootenvSchema", ChrootenvSchema},
16+
{"OsConfigSchema", OsConfigSchema},
17+
}
18+
19+
for _, tt := range tests {
20+
t.Run(tt.name, func(t *testing.T) {
21+
if len(tt.schema) == 0 {
22+
t.Fatalf("%s: expected non-empty bytes, got empty", tt.name)
23+
}
24+
})
25+
}
26+
}
27+
28+
func TestEmbeddedSchemasValidJSON(t *testing.T) {
29+
tests := []struct {
30+
name string
31+
schema []byte
32+
}{
33+
{"ImageTemplateSchema", ImageTemplateSchema},
34+
{"ConfigSchema", ConfigSchema},
35+
{"ChrootenvSchema", ChrootenvSchema},
36+
{"OsConfigSchema", OsConfigSchema},
37+
}
38+
39+
for _, tt := range tests {
40+
t.Run(tt.name, func(t *testing.T) {
41+
var doc map[string]any
42+
if err := json.Unmarshal(tt.schema, &doc); err != nil {
43+
t.Fatalf("%s: invalid JSON: %v", tt.name, err)
44+
}
45+
})
46+
}
47+
}
48+
49+
func TestEmbeddedSchemasHaveMetadataFields(t *testing.T) {
50+
tests := []struct {
51+
name string
52+
schema []byte
53+
wantSchema string
54+
wantID string
55+
}{
56+
{
57+
name: "ImageTemplateSchema",
58+
schema: ImageTemplateSchema,
59+
wantSchema: "https://json-schema.org/draft/2020-12/schema",
60+
wantID: "os-image-template.schema.json",
61+
},
62+
{
63+
name: "ConfigSchema",
64+
schema: ConfigSchema,
65+
wantSchema: "https://json-schema.org/draft/2020-12/schema",
66+
wantID: "https://github.com/open-edge-platform/os-image-composer/schemas/os-image-composer-config.schema.json",
67+
},
68+
{
69+
name: "ChrootenvSchema",
70+
schema: ChrootenvSchema,
71+
wantSchema: "https://json-schema.org/draft/2020-12/schema",
72+
wantID: "chrootenv-config.schema.json",
73+
},
74+
{
75+
name: "OsConfigSchema",
76+
schema: OsConfigSchema,
77+
wantSchema: "https://json-schema.org/draft/2020-12/schema",
78+
wantID: "os-config.schema.json",
79+
},
80+
}
81+
82+
for _, tt := range tests {
83+
t.Run(tt.name, func(t *testing.T) {
84+
var doc map[string]any
85+
if err := json.Unmarshal(tt.schema, &doc); err != nil {
86+
t.Fatalf("unexpected JSON parse error: %v", err)
87+
}
88+
89+
gotSchema, _ := doc["$schema"].(string)
90+
if gotSchema != tt.wantSchema {
91+
t.Errorf("$schema: got %q, want %q", gotSchema, tt.wantSchema)
92+
}
93+
94+
gotID, _ := doc["$id"].(string)
95+
if gotID != tt.wantID {
96+
t.Errorf("$id: got %q, want %q", gotID, tt.wantID)
97+
}
98+
})
99+
}
100+
}

‎internal/config/schema/os-image-template.schema.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,11 @@
310310
"description": "Repository base URL",
311311
"format": "uri"
312312
},
313+
"path": {
314+
"type": "string",
315+
"description": "Local directory path for file-based repositories",
316+
"pattern": "^/"
317+
},
313318
"pkey": {
314319
"type": "string",
315320
"description": "Public GPG key URL for package verification or [trusted=yes] to skip verification",
@@ -372,7 +377,10 @@
372377
}
373378
}
374379
},
375-
"required": ["codename", "url"],
380+
"oneOf": [
381+
{ "required": ["codename", "path"] },
382+
{ "required": ["codename", "url"] }
383+
],
376384
"anyOf": [
377385
{ "required": ["pkey"] },
378386
{ "required": ["pkeys"] }

0 commit comments

Comments
 (0)