Respecting always_run from config file

Author: Hector Vido

cmd/ci-operator-prowgen/README.md

@@ -0,0 +1,34 @@
+prowgen
+=======
+
+Prowgen is a tool that converts [ci-operator configuration](https://docs.ci.openshift.org/docs/architecture/ci-operator/)
+into [Prowjobs](https://docs.prow.k8s.io/docs/jobs/).
+
+The objetive is to made `ci-operator configuration` the only source of truth for jobs created
+by **prowgen**, based on configurations inside `openshift/ci-operator/config/`, replacing any
+user manual modification directly on `openshift/ci-operator/jobs/`.
+
+Prowgen is normally executed by `make update/make jobs` inside `openshift/release` folder. 
+
+Testing
+-------
+
+`Prowgen` is hardcoded to use `GOPATH` + `src/github.com/openshift/release`, if you want to test it on your machine
+you can use a symbolic link ponting to your `openshift/release` clone:
+
+```bash
+# generally GOPATH=~/go
+ln -s ~/cloned-repos/openshift/release ~/go/src/github.com/openshift/release
+```
+
+Then you can execute `ci-operator-prowgen`:
+
+```bash
+ci-operator-prowgen \
+    --from-release-repo \
+    --to-release-repo \
+    --known-infra-file infra-build-farm-periodics.yaml \
+    --known-infra-file infra-periodics.yaml \
+    --known-infra-file infra-image-mirroring.yaml \
+    --known-infra-file infra-periodics-origin-release-images.yaml
+```

cmd/ci-operator-prowgen/main.go

@@ -195,8 +195,8 @@ func writeToDir(dir string, c map[string]*prowconfig.JobConfig) error {
 	errCh := make(chan error)
 	map_ := func() error {
 		for x := range ch {
-			i := strings.Index(x.k, "/")
-			org, repo := x.k[:i], x.k[i+1:]
+			i := strings.Split(x.k, "/")
+			org, repo := i[0], i[1]
 			if err := jc.WriteToDir(dir, org, repo, x.v, prowgen.Generator, nil); err != nil {
 				errCh <- err
 			}

pkg/jobconfig/files.go

@@ -365,7 +365,7 @@ func WriteToDir(jobDir, org, repo string, jobConfig *prowconfig.JobConfig, gener
 	return nil
 }
 
-// Given two JobConfig, merge jobs from the `source` one to to `destination`
+// Given two JobConfig, merge jobs from the `source` one to `destination`
 // one. Jobs are matched by name. All jobs from `source` will be present in
 // `destination` - if there were jobs with the same name in `destination`, they
 // will be updated. All jobs in `destination` that are not overwritten this
@@ -471,9 +471,6 @@ func mergeJobConfig(destination, source *prowconfig.JobConfig, allJobs sets.Set[
 func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 	merged := *new
 
-	merged.AlwaysRun = old.AlwaysRun
-	merged.RunIfChanged = old.RunIfChanged
-	merged.SkipIfOnlyChanged = old.SkipIfOnlyChanged
 	merged.MaxConcurrency = old.MaxConcurrency
 	merged.SkipReport = old.SkipReport
 	merged.Cluster = func() string {
@@ -485,20 +482,23 @@ func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 		return ""
 	}()
 
-	if new.RunIfChanged != "" || new.SkipIfOnlyChanged != "" || new.Annotations["pipeline_run_if_changed"] != "" {
-		merged.RunIfChanged = new.RunIfChanged
-		merged.SkipIfOnlyChanged = new.SkipIfOnlyChanged
-		merged.AlwaysRun = new.AlwaysRun
-	}
-
-	// TODO(muller): Special case images jobs for now. Some repos are marking
-	// images jobs as optional for which we do not have syntax in ci-operator (should we?).
-	// Tolerate manual changes for these jobs for now
+   // TODO(muller): Special case images jobs for now. Some repos are marking
+   // images jobs as optional for which we do not have syntax in ci-operator (should we?).
+   // Tolerate manual changes for these jobs for now
+   // TODO(hector): Image jobs should keep considering always_run for now.
 	if strings.HasSuffix(merged.Name, "-images") {
 		merged.Optional = old.Optional
 		if new.Optional {
 			merged.Optional = new.Optional
 		}
+		merged.RunIfChanged = old.RunIfChanged
+		merged.SkipIfOnlyChanged = old.SkipIfOnlyChanged
+		merged.AlwaysRun = old.AlwaysRun
+		if new.RunIfChanged != "" || new.SkipIfOnlyChanged != "" || new.Annotations["pipeline_run_if_changed"] != "" {
+			merged.RunIfChanged = new.RunIfChanged
+			merged.SkipIfOnlyChanged = new.SkipIfOnlyChanged
+			merged.AlwaysRun = new.AlwaysRun
+		}
 	}
 
 	return merged

pkg/prowgen/prowgen.go

@@ -245,6 +245,10 @@ type generatePresubmitOptions struct {
 	disableRehearsal     bool
 }
 
+func (opts *generatePresubmitOptions) shouldAlwaysRun() bool {
+	return opts.runIfChanged == "" && opts.skipIfOnlyChanged == "" && !opts.defaultDisable && opts.pipelineRunIfChanged == ""
+}
+
 type generatePresubmitOption func(options *generatePresubmitOptions)
 
 func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, info *ProwgenInfo, options ...generatePresubmitOption) *prowconfig.Presubmit {
@@ -269,7 +273,7 @@ func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, i
 	}
 	pj := &prowconfig.Presubmit{
 		JobBase:   base,
-		AlwaysRun: opts.runIfChanged == "" && opts.skipIfOnlyChanged == "" && !opts.defaultDisable && opts.pipelineRunIfChanged == "",
+		AlwaysRun: opts.shouldAlwaysRun(),
 		Brancher:  prowconfig.Brancher{Branches: sets.List(sets.New[string](jc.ExactlyBranch(info.Branch), jc.FeatureBranch(info.Branch)))},
 		Reporter: prowconfig.Reporter{
 			Context: fmt.Sprintf("ci/prow/%s", shortName),

test/integration/ci-operator-prowgen/input/jobs/super/duper/super-duper-master-presubmits.yaml

@@ -1,17 +1,15 @@
 presubmits:
   super/duper:
   - agent: kubernetes
-    always_run: false
+    always_run: true
     branches:
     - ^master$
     - ^master-
     context: ci/prow/images
     decorate: true
     max_concurrency: 100
     name: pull-ci-super-duper-master-images
-    optional: true
     rerun_command: /test images
-    run_if_changed: changes
     skip_cloning: true
     skip_report: true
     spec:

test/integration/ci-operator-prowgen/output/jobs/super/duper/super-duper-master-presubmits.yaml

@@ -180,7 +180,7 @@ presubmits:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e,?($|\s.*)
   - agent: kubernetes
-    always_run: false
+    always_run: true
     branches:
     - ^master$
     - ^master-
@@ -193,9 +193,7 @@ presubmits:
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     max_concurrency: 100
     name: pull-ci-super-duper-master-images
-    optional: true
     rerun_command: /test images
-    run_if_changed: changes
     skip_report: true
     spec:
       containers: