From 1801c43ae9e61e9542cca0f8aa851b4165947671 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Thu, 16 Jan 2025 00:33:07 +0400 Subject: [PATCH 01/11] TBT-137 Be able to use environment variables across different repos --- .../scheduler/record/account_env_vars.rb | 5 +++++ lib/travis/scheduler/serialize/worker.rb | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lib/travis/scheduler/record/account_env_vars.rb diff --git a/lib/travis/scheduler/record/account_env_vars.rb b/lib/travis/scheduler/record/account_env_vars.rb new file mode 100644 index 00000000..1bd60311 --- /dev/null +++ b/lib/travis/scheduler/record/account_env_vars.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class AccountEnvVars < ActiveRecord::Base + belongs_to :owner, polymorphic: true +end diff --git a/lib/travis/scheduler/serialize/worker.rb b/lib/travis/scheduler/serialize/worker.rb index b4239ff6..e43f502f 100644 --- a/lib/travis/scheduler/serialize/worker.rb +++ b/lib/travis/scheduler/serialize/worker.rb @@ -251,6 +251,24 @@ def env_vars_with_custom_keys job.env_vars + custom_keys end + def env_vars_with_account_vars + info "Mapped account env vars: #{account_env_vars}" + final_vars = env_vars_with_custom_keys + account_env_vars + info "Merged env vars: #{final_vars}" + final_vars + end + + def account_env_vars + info "Fetching account env vars for owner: #{build.sender_id} with owner type: #{build.owner_type}" + vars = AccountEnvVars.where(owner_id: build.owner_id, owner_type: build.owner_type) + info "Results for owner: #{build.owner_id}, variables: #{vars}" + vars.map { |var| env_var(var) } + end + + def env_var(var) + { name: var.name, value: var.value.decrypt, public: var.public, branch: nil } + end + def custom_keys return [] if job.decrypted_config[:keys].blank? From d0c0e32e9b6bd5b27f5eb55c78e7745ef6235e51 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Thu, 16 Jan 2025 23:13:46 +0400 Subject: [PATCH 02/11] TBT-137 Set env_vars_with_account_vars as env_vars --- lib/travis/scheduler/serialize/worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/scheduler/serialize/worker.rb b/lib/travis/scheduler/serialize/worker.rb index e43f502f..3a630633 100644 --- a/lib/travis/scheduler/serialize/worker.rb +++ b/lib/travis/scheduler/serialize/worker.rb @@ -21,7 +21,7 @@ def data vm_size: job.vm_size, queue: job.queue, config: job.decrypted_config, - env_vars: env_vars_with_custom_keys, + env_vars: env_vars_with_account_vars, job: job_data, host: Travis::Scheduler.config.host, source: build_data, From a17879ed30eb29730dcb9db9cf13137345e12651 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Thu, 16 Jan 2025 23:35:39 +0400 Subject: [PATCH 03/11] TBT-137 Fix logger usage --- lib/travis/scheduler/serialize/worker.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/travis/scheduler/serialize/worker.rb b/lib/travis/scheduler/serialize/worker.rb index 3a630633..ef9f9b08 100644 --- a/lib/travis/scheduler/serialize/worker.rb +++ b/lib/travis/scheduler/serialize/worker.rb @@ -252,16 +252,16 @@ def env_vars_with_custom_keys end def env_vars_with_account_vars - info "Mapped account env vars: #{account_env_vars}" + Travis.logger.info "Mapped account env vars: #{account_env_vars}" final_vars = env_vars_with_custom_keys + account_env_vars - info "Merged env vars: #{final_vars}" + Travis.logger.info "Merged env vars: #{final_vars}" final_vars end def account_env_vars - info "Fetching account env vars for owner: #{build.sender_id} with owner type: #{build.owner_type}" + Travis.logger.info "Fetching account env vars for owner: #{build.sender_id} with owner type: #{build.owner_type}" vars = AccountEnvVars.where(owner_id: build.owner_id, owner_type: build.owner_type) - info "Results for owner: #{build.owner_id}, variables: #{vars}" + Travis.logger.info "Results for owner: #{build.owner_id}, variables: #{vars}" vars.map { |var| env_var(var) } end From f2f815636ac5837acf504df759aae63d5eb5f17b Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 00:03:58 +0400 Subject: [PATCH 04/11] TBT-137 Add missing account_env_vars to record.rb --- lib/travis/scheduler/record.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/scheduler/record.rb b/lib/travis/scheduler/record.rb index b4b61910..b764390e 100644 --- a/lib/travis/scheduler/record.rb +++ b/lib/travis/scheduler/record.rb @@ -21,3 +21,4 @@ require 'travis/scheduler/record/subscription' require 'travis/scheduler/record/trial' require 'travis/scheduler/record/user' +require 'travis/scheduler/record/account_env_vars' From 398319a81798c07b74add8a92ed3a3f3ba5979c4 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 01:32:50 +0400 Subject: [PATCH 05/11] TBT-137 Move account env var processing to job.rb --- lib/travis/scheduler/serialize/worker.rb | 20 +------------------- lib/travis/scheduler/serialize/worker/job.rb | 11 +++++++++++ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/travis/scheduler/serialize/worker.rb b/lib/travis/scheduler/serialize/worker.rb index ef9f9b08..b4239ff6 100644 --- a/lib/travis/scheduler/serialize/worker.rb +++ b/lib/travis/scheduler/serialize/worker.rb @@ -21,7 +21,7 @@ def data vm_size: job.vm_size, queue: job.queue, config: job.decrypted_config, - env_vars: env_vars_with_account_vars, + env_vars: env_vars_with_custom_keys, job: job_data, host: Travis::Scheduler.config.host, source: build_data, @@ -251,24 +251,6 @@ def env_vars_with_custom_keys job.env_vars + custom_keys end - def env_vars_with_account_vars - Travis.logger.info "Mapped account env vars: #{account_env_vars}" - final_vars = env_vars_with_custom_keys + account_env_vars - Travis.logger.info "Merged env vars: #{final_vars}" - final_vars - end - - def account_env_vars - Travis.logger.info "Fetching account env vars for owner: #{build.sender_id} with owner type: #{build.owner_type}" - vars = AccountEnvVars.where(owner_id: build.owner_id, owner_type: build.owner_type) - Travis.logger.info "Results for owner: #{build.owner_id}, variables: #{vars}" - vars.map { |var| env_var(var) } - end - - def env_var(var) - { name: var.name, value: var.value.decrypt, public: var.public, branch: nil } - end - def custom_keys return [] if job.decrypted_config[:keys].blank? diff --git a/lib/travis/scheduler/serialize/worker/job.rb b/lib/travis/scheduler/serialize/worker/job.rb index 13a21379..f26bf2f7 100644 --- a/lib/travis/scheduler/serialize/worker/job.rb +++ b/lib/travis/scheduler/serialize/worker/job.rb @@ -18,6 +18,17 @@ def env_vars vars = repository.settings.env_vars vars = vars.public unless secure_env? vars.map { |var| env_var(var) } + Travis.logger.info "Mapped account env vars: #{account_env_vars}" + final_vars = vars + account_env_vars + Travis.logger.info "Merged env vars: #{final_vars}" + final_vars + end + + def account_env_vars + Travis.logger.info "Fetching account env vars for owner: #{job.sender_id} with owner type: #{job.owner_type}" + vars = AccountEnvVars.where(owner_id: job.owner_id, owner_type: job.owner_type) + Travis.logger.info "Results for owner: #{job.owner_id}, variables: #{vars}" + vars.map { |var| env_var(var) } end def secure_env? From 6ce25c457ba78a6ee1ee8e7edad3bd187c2a2642 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 01:49:28 +0400 Subject: [PATCH 06/11] TBT-137 Fix wrong field name --- lib/travis/scheduler/serialize/worker/job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/scheduler/serialize/worker/job.rb b/lib/travis/scheduler/serialize/worker/job.rb index f26bf2f7..2b1c7296 100644 --- a/lib/travis/scheduler/serialize/worker/job.rb +++ b/lib/travis/scheduler/serialize/worker/job.rb @@ -25,7 +25,7 @@ def env_vars end def account_env_vars - Travis.logger.info "Fetching account env vars for owner: #{job.sender_id} with owner type: #{job.owner_type}" + Travis.logger.info "Fetching account env vars for owner: #{job.owner_id} with owner type: #{job.owner_type}" vars = AccountEnvVars.where(owner_id: job.owner_id, owner_type: job.owner_type) Travis.logger.info "Results for owner: #{job.owner_id}, variables: #{vars}" vars.map { |var| env_var(var) } From 2e169f57f05121f7fb580f81d55f363c00d1aae5 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 04:07:14 +0400 Subject: [PATCH 07/11] TBT-137 Add encryption for env value --- lib/travis/scheduler/record/account_env_vars.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/scheduler/record/account_env_vars.rb b/lib/travis/scheduler/record/account_env_vars.rb index 1bd60311..012cc148 100644 --- a/lib/travis/scheduler/record/account_env_vars.rb +++ b/lib/travis/scheduler/record/account_env_vars.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true class AccountEnvVars < ActiveRecord::Base + serialize :value, Travis::EncryptedColumn.new belongs_to :owner, polymorphic: true end From cb7bea7f25f69e488650f481d727461acd217842 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 04:37:11 +0400 Subject: [PATCH 08/11] TBT-137 remove model decrypt serialization --- lib/travis/scheduler/record/account_env_vars.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/travis/scheduler/record/account_env_vars.rb b/lib/travis/scheduler/record/account_env_vars.rb index 012cc148..1bd60311 100644 --- a/lib/travis/scheduler/record/account_env_vars.rb +++ b/lib/travis/scheduler/record/account_env_vars.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true class AccountEnvVars < ActiveRecord::Base - serialize :value, Travis::EncryptedColumn.new belongs_to :owner, polymorphic: true end From 02b5feb65bffee7699b8e9e44d6af31bdf8bf2a6 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 11:03:33 +0400 Subject: [PATCH 09/11] TBT-137 add model decrypt serialization --- lib/travis/scheduler/record/account_env_vars.rb | 1 + lib/travis/scheduler/serialize/worker/job.rb | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/travis/scheduler/record/account_env_vars.rb b/lib/travis/scheduler/record/account_env_vars.rb index 1bd60311..012cc148 100644 --- a/lib/travis/scheduler/record/account_env_vars.rb +++ b/lib/travis/scheduler/record/account_env_vars.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true class AccountEnvVars < ActiveRecord::Base + serialize :value, Travis::EncryptedColumn.new belongs_to :owner, polymorphic: true end diff --git a/lib/travis/scheduler/serialize/worker/job.rb b/lib/travis/scheduler/serialize/worker/job.rb index 2b1c7296..e5770f10 100644 --- a/lib/travis/scheduler/serialize/worker/job.rb +++ b/lib/travis/scheduler/serialize/worker/job.rb @@ -28,7 +28,7 @@ def account_env_vars Travis.logger.info "Fetching account env vars for owner: #{job.owner_id} with owner type: #{job.owner_type}" vars = AccountEnvVars.where(owner_id: job.owner_id, owner_type: job.owner_type) Travis.logger.info "Results for owner: #{job.owner_id}, variables: #{vars}" - vars.map { |var| env_var(var) } + vars.map { |var| account_env_var(var) } end def secure_env? @@ -95,6 +95,11 @@ def env_var(var) { name: var.name, value: var.value.decrypt, public: var.public, branch: var.branch } end + def account_env_var(var) + { name: var.name, value: var.value, public: var.public, branch: nil } + end + + def has_secure_vars?(key) job.config.key?(key) && job.config[key].respond_to?(:key?) && From e0cd0092329b359cd17a9efc6edd197f7549bdc5 Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 11:52:36 +0400 Subject: [PATCH 10/11] TBT-137 Fix issue with env merge --- lib/travis/scheduler/serialize/worker/job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/travis/scheduler/serialize/worker/job.rb b/lib/travis/scheduler/serialize/worker/job.rb index e5770f10..f508583e 100644 --- a/lib/travis/scheduler/serialize/worker/job.rb +++ b/lib/travis/scheduler/serialize/worker/job.rb @@ -17,9 +17,9 @@ class Job < Struct.new(:job, :config) def env_vars vars = repository.settings.env_vars vars = vars.public unless secure_env? - vars.map { |var| env_var(var) } + mapped_vars = vars.map { |var| env_var(var) } Travis.logger.info "Mapped account env vars: #{account_env_vars}" - final_vars = vars + account_env_vars + final_vars = mapped_vars + account_env_vars Travis.logger.info "Merged env vars: #{final_vars}" final_vars end From 0bec91185d6f676fc57ee8d18d81b10e0f73100d Mon Sep 17 00:00:00 2001 From: tonoyansergey-devt Date: Fri, 17 Jan 2025 12:26:44 +0400 Subject: [PATCH 11/11] TBT-137 Use unique keys to exclude duplicate vars --- lib/travis/scheduler/serialize/worker/job.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/travis/scheduler/serialize/worker/job.rb b/lib/travis/scheduler/serialize/worker/job.rb index f508583e..29be6aeb 100644 --- a/lib/travis/scheduler/serialize/worker/job.rb +++ b/lib/travis/scheduler/serialize/worker/job.rb @@ -17,10 +17,20 @@ class Job < Struct.new(:job, :config) def env_vars vars = repository.settings.env_vars vars = vars.public unless secure_env? + mapped_vars = vars.map { |var| env_var(var) } - Travis.logger.info "Mapped account env vars: #{account_env_vars}" - final_vars = mapped_vars + account_env_vars + + account_vars = account_env_vars + Travis.logger.info "Mapped account env vars: #{account_vars}" + + repo_var_hash = mapped_vars.map { |v| [v[:name], v] }.to_h + account_var_hash = account_vars.map { |v| [v[:name], v] }.to_h + + final_vars_hash = repo_var_hash.merge(account_var_hash) + + final_vars = final_vars_hash.values Travis.logger.info "Merged env vars: #{final_vars}" + final_vars end