Respecting always_run from config file

Author: Hector Vido

cmd/ci-operator-prowgen/README.md

@@ -0,0 +1,34 @@
+prowgen
+=======
+
+Prowgen is a tool that converts [ci-operator configuration](https://docs.ci.openshift.org/docs/architecture/ci-operator/)
+into [Prowjobs](https://docs.prow.k8s.io/docs/jobs/).
+
+The objetive is to made `ci-operator configuration` the only source of truth for jobs created
+by **prowgen**, based on configurations inside `openshift/ci-operator/config/`, replacing any
+user manual modification directly on `openshift/ci-operator/jobs/`.
+
+Prowgen is normally executed by `make update/make jobs` inside `openshift/release` folder. 
+
+Testing
+-------
+
+`Prowgen` is hardcoded to use `GOPATH` + `src/github.com/openshift/release`, if you want to test it on your machine
+you can use a symbolic link ponting to your `openshift/release` clone:
+
+```bash
+# generally GOPATH=~/go
+ln -s ~/cloned-repos/openshift/release ~/go/src/github.com/openshift/release
+```
+
+Then you can execute `ci-operator-prowgen`:
+
+```bash
+ci-operator-prowgen \
+    --from-release-repo \
+    --to-release-repo \
+    --known-infra-file infra-build-farm-periodics.yaml \
+    --known-infra-file infra-periodics.yaml \
+    --known-infra-file infra-image-mirroring.yaml \
+    --known-infra-file infra-periodics-origin-release-images.yaml
+```

cmd/ci-operator-prowgen/main.go

@@ -159,6 +159,9 @@ func generateJobs(resolver registry.Resolver, cache map[string]*config.Prowgen,
 			configSpec = &resolved
 		}
 		generated, err := prowgen.GenerateJobs(configSpec, pInfo)
+		if orgRepo == "stolostron/policy-collection" {
+			print(orgRepo)
+		}
 		if err != nil {
 			return err
 		}
@@ -195,8 +198,8 @@ func writeToDir(dir string, c map[string]*prowconfig.JobConfig) error {
 	errCh := make(chan error)
 	map_ := func() error {
 		for x := range ch {
-			i := strings.Index(x.k, "/")
-			org, repo := x.k[:i], x.k[i+1:]
+			i := strings.Split(x.k, "/")
+			org, repo := i[0], i[1]
 			if err := jc.WriteToDir(dir, org, repo, x.v, prowgen.Generator, nil); err != nil {
 				errCh <- err
 			}

pkg/jobconfig/files.go

@@ -365,7 +365,7 @@ func WriteToDir(jobDir, org, repo string, jobConfig *prowconfig.JobConfig, gener
 	return nil
 }
 
-// Given two JobConfig, merge jobs from the `source` one to to `destination`
+// Given two JobConfig, merge jobs from the `source` one to `destination`
 // one. Jobs are matched by name. All jobs from `source` will be present in
 // `destination` - if there were jobs with the same name in `destination`, they
 // will be updated. All jobs in `destination` that are not overwritten this
@@ -471,9 +471,6 @@ func mergeJobConfig(destination, source *prowconfig.JobConfig, allJobs sets.Set[
 func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 	merged := *new
 
-	merged.AlwaysRun = old.AlwaysRun
-	merged.RunIfChanged = old.RunIfChanged
-	merged.SkipIfOnlyChanged = old.SkipIfOnlyChanged
 	merged.MaxConcurrency = old.MaxConcurrency
 	merged.SkipReport = old.SkipReport
 	merged.Cluster = func() string {
@@ -485,12 +482,6 @@ func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 		return ""
 	}()
 
-	if new.RunIfChanged != "" || new.SkipIfOnlyChanged != "" || new.Annotations["pipeline_run_if_changed"] != "" {
-		merged.RunIfChanged = new.RunIfChanged
-		merged.SkipIfOnlyChanged = new.SkipIfOnlyChanged
-		merged.AlwaysRun = new.AlwaysRun
-	}
-
 	// TODO(muller): Special case images jobs for now. Some repos are marking
 	// images jobs as optional for which we do not have syntax in ci-operator (should we?).
 	// Tolerate manual changes for these jobs for now
@@ -499,6 +490,9 @@ func mergePresubmits(old, new *prowconfig.Presubmit) prowconfig.Presubmit {
 		if new.Optional {
 			merged.Optional = new.Optional
 		}
+		if merged.Optional {
+			merged.AlwaysRun = false
+		}
 	}
 
 	return merged

pkg/prowgen/prowgen.go

@@ -132,7 +132,18 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro
 		injectArchitectureLabels(jobBaseGen, configSpec.Images)
 
 		jobBaseGen.PodSpec.Add(Targets(presubmitTargets...))
-		presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(jobBaseGen, imagesTestName, info))
+		presubmits[orgrepo] = append(presubmits[orgrepo], *generatePresubmitForTest(jobBaseGen, imagesTestName, info, func(options *generatePresubmitOptions) {
+			if len(configSpec.Images) > 0 {
+				optional := false
+				for _, config := range configSpec.Images {
+					if config.Optional {
+						optional = true
+						break
+					}
+				}
+				options.optional = optional
+			}
+		}))
 
 		if configSpec.PromotionConfiguration != nil {
 			jobBaseGen = newJobBaseBuilder().TestName(imagesTestName)
@@ -245,6 +256,10 @@ type generatePresubmitOptions struct {
 	disableRehearsal     bool
 }
 
+func (opts *generatePresubmitOptions) shouldAlwaysRun() bool {
+	return opts.runIfChanged == "" && opts.skipIfOnlyChanged == "" && !opts.defaultDisable && opts.pipelineRunIfChanged == ""
+}
+
 type generatePresubmitOption func(options *generatePresubmitOptions)
 
 func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, info *ProwgenInfo, options ...generatePresubmitOption) *prowconfig.Presubmit {
@@ -269,7 +284,7 @@ func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, i
 	}
 	pj := &prowconfig.Presubmit{
 		JobBase:   base,
-		AlwaysRun: opts.runIfChanged == "" && opts.skipIfOnlyChanged == "" && !opts.defaultDisable && opts.pipelineRunIfChanged == "",
+		AlwaysRun: opts.shouldAlwaysRun(),
 		Brancher:  prowconfig.Brancher{Branches: sets.List(sets.New[string](jc.ExactlyBranch(info.Branch), jc.FeatureBranch(info.Branch)))},
 		Reporter: prowconfig.Reporter{
 			Context: fmt.Sprintf("ci/prow/%s", shortName),