Skip to content

Commit 7da072b

Browse files
alvarocabanasalvarocabanas
committed
feat: add staging e2e on demand test using the large runners
1 parent 458fc49 commit 7da072b

File tree

21 files changed

+330
-23
lines changed

21 files changed

+330
-23
lines changed

‎.github/workflows/on_demand_staging_e2e.yml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
name: 🍯 E2E pointing to staging
2+
3+
on:
4+
push:
5+
6+
permissions:
7+
contents: read
8+
packages: write
9+
10+
# NOTE: The below environment vars are bumped automatically by Renovate so we always use the latest
11+
# versions of the packages in our E2E tests. If you change the names or move these values outside
12+
# the GitHub workflows please update `renovate.json` accordingly to avoid breaking the automation!
13+
env:
14+
HOST_E2E_INFRA_AGENT_VERSION: "1.72.4"
15+
HOST_E2E_NRDOT_VERSION: "1.8.0"
16+
17+
jobs:
18+
build-packages:
19+
name: Build packages
20+
uses: ./.github/workflows/component_packages.yml
21+
with:
22+
pre-release: false
23+
skip_sign: true
24+
# tag for staging
25+
tag_name: 0.600.${{ github.run_id }}
26+
27+
28+
on-host-linux-e2e:
29+
name: OnHost Linux E2E tests
30+
needs: [ build-packages ]
31+
runs-on: linux-4-core-nr-control
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
group:
36+
- ebpf-agent
37+
- remote-config
38+
- infra-agent
39+
- nrdot-agent
40+
- proxy
41+
steps:
42+
- name: Record job start time
43+
run: echo "JOB_START_TIME=$(date +%s)" >> "$GITHUB_ENV"
44+
45+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
46+
- uses: ./.github/actions/install-rust-toolchain
47+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
48+
with:
49+
name: built-binaries-0.600.${{ github.run_id }}
50+
path: ./
51+
- name: Build e2e-runner binary
52+
run: cargo build -p e2e-runner
53+
- name: "Linux e2e ${{ matrix.group }}"
54+
run: |
55+
echo "${{ secrets.AC_FC_STAG_E2E_NR_SYSTEM_IDENTITY_PRIVATE_KEY }}" > /tmp/newrelic-agent-control-priv.key
56+
sudo ${{ github.workspace }}/target/debug/e2e-runner --log-level=debug ${{ matrix.group }} \
57+
--artifacts-package-dir ./dist \
58+
--nr-api-key "${{ secrets.AC_FC_STAG_E2E_API_KEY }}" \
59+
--nr-license-key "${{ secrets.AC_FC_STAG_E2E_LICENSE_KEY }}" \
60+
--nr-account-id "${{ secrets.AC_FC_STAG_E2E_ACCOUNT_ID }}" \
61+
--nr-region "staging" \
62+
--system-identity-client-id "${{ secrets.AC_FC_STAG_E2E_NR_SYSTEM_IDENTITY_CLIENT_ID }}" \
63+
--agent-control-private-key "/tmp/newrelic-agent-control-priv.key" \
64+
--agent-control-version "0.600.${{ github.run_id }}" \
65+
--recipes-repo-branch "main" \
66+
--infra-agent-version "${{ env.HOST_E2E_INFRA_AGENT_VERSION }}" \
67+
--nrdot-version "${{ env.HOST_E2E_NRDOT_VERSION }}"
68+
69+
- name: Report test result to New Relic
70+
if: always()
71+
env:
72+
NR_LICENSE_KEY: ${{ secrets.AC_FC_STAG_E2E_LICENSE_KEY }}
73+
NR_ACCOUNT_ID: ${{ secrets.AC_FC_STAG_E2E_ACCOUNT_ID }}
74+
E2E_SCENARIO: ${{ matrix.group }}
75+
E2E_ENVIRONMENT: linux
76+
E2E_STATUS: ${{ job.status }}
77+
E2E_CALLER_WORKFLOW: on_demand_staging_e2e
78+
run: bash .github/workflows/scripts/report_e2e_result.sh
79+
80+
- name: Upload e2e result summary
81+
if: always()
82+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
83+
with:
84+
name: e2e-result-linux-${{ matrix.group }}
85+
path: e2e-result-*-linux-${{ matrix.group }}.txt
86+
if-no-files-found: warn
87+
88+
k8s-e2e-tests:
89+
name: K8s e2e tests ${{ matrix.group }}
90+
runs-on: linux-4-core-nr-control
91+
strategy:
92+
fail-fast: false
93+
matrix:
94+
group:
95+
# apm operator doesn't allow pointing to staging
96+
- collector
97+
- fleet-control
98+
- ebpf
99+
- dynamic
100+
- custom-repo
101+
# proxy has issues pointing to staging
102+
steps:
103+
- name: Record job start time
104+
run: echo "JOB_START_TIME=$(date +%s)" >> "$GITHUB_ENV"
105+
106+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
107+
108+
- name: Free up space
109+
run: bash .github/workflows/scripts/space_cleanup.sh
110+
111+
- name: Setup Minikube
112+
uses: manusa/actions-setup-minikube@96202dee4ae1c2f46a62fe197273aaf22b83f42d # v2.16.1
113+
with:
114+
minikube version: v1.35.0
115+
kubernetes version: "v1.30.14"
116+
start args: '--cni=calico'
117+
driver: docker
118+
github token: ${{ secrets.GITHUB_TOKEN }}
119+
120+
- name: Install Tilt
121+
run: |
122+
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
123+
124+
- uses: ./.github/actions/install-rust-toolchain
125+
with:
126+
targets: >-
127+
x86_64-unknown-linux-musl
128+
129+
- uses: ./.github/actions/rust-cache
130+
with:
131+
identifier: 'test-dev'
132+
restore-strategy: 'nearest'
133+
save-cache: false
134+
135+
- name: Install Zig
136+
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
137+
138+
- uses: taiki-e/install-action@cbb1dcaa26e1459e2876c39f61c1e22a1258aac5 # v2.68.33
139+
with:
140+
tool: cargo-zigbuild@0.20.1
141+
142+
- name: Run k8s e2e-test ${{ matrix.group }}
143+
uses: newrelic/newrelic-integration-e2e-action@c548a33a0c2941a4db4899ded766923eb3178e0e # v1
144+
env:
145+
# SI L2->L2 for AC
146+
NR_SYSTEM_IDENTITY_CLIENT_ID: ${{ secrets.AC_FC_STAG_E2E_NR_SYSTEM_IDENTITY_CLIENT_ID }}
147+
NR_SYSTEM_IDENTITY_PRIVATE_KEY: ${{ secrets.AC_FC_STAG_E2E_NR_SYSTEM_IDENTITY_PRIVATE_KEY }}
148+
USE_LATEST_FLUX: false
149+
# Tilt configs
150+
ARCH: amd64
151+
with:
152+
retry_attempts: 40
153+
retry_seconds: 15
154+
agent_enabled: false
155+
spec_path: test/k8s-e2e/${{ matrix.group }}/e2e-${{ matrix.group }}.yml
156+
account_id: ${{ secrets.AC_FC_STAG_E2E_ACCOUNT_ID }}
157+
api_key: ${{ secrets.AC_FC_STAG_E2E_API_KEY }}
158+
license_key: ${{ secrets.AC_FC_STAG_E2E_LICENSE_KEY }}
159+
region: Staging
160+
161+
- name: Report test result to New Relic
162+
if: always()
163+
env:
164+
NR_LICENSE_KEY: ${{ secrets.AC_FC_STAG_E2E_LICENSE_KEY }}
165+
NR_ACCOUNT_ID: ${{ secrets.AC_FC_STAG_E2E_ACCOUNT_ID }}
166+
E2E_REGION: staging
167+
E2E_SCENARIO: ${{ matrix.group }}
168+
E2E_ENVIRONMENT: k8s
169+
E2E_STATUS: ${{ job.status }}
170+
E2E_CALLER_WORKFLOW: on_demand_staging_e2e
171+
run: bash .github/workflows/scripts/report_e2e_result.sh
172+
173+
- name: Upload e2e result summary
174+
if: always()
175+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
176+
with:
177+
name: e2e-result-k8s-${{ matrix.group }}
178+
path: e2e-result-*-k8s-${{ matrix.group }}.txt
179+
if-no-files-found: warn
180+
181+
k8s-e2e-tests-check: # The job is useful to notify the failure
182+
name: K8s e2e tests
183+
if: always() # cannot be skipped even if required steps fail
184+
runs-on: ubuntu-latest
185+
needs: [ k8s-e2e-tests ]
186+
steps:
187+
- name: Check if needed jobs succeeded
188+
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # 05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
189+
with:
190+
jobs: ${{ toJSON(needs) }}

‎.github/workflows/scripts/report_e2e_result.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
# E2E_ENVIRONMENT - One of: linux, windows, k8s
88
# E2E_STATUS - job.status value: success, failure, or cancelled
99
# E2E_CALLER_WORKFLOW - Name of the top-level workflow that triggered this run
10+
# E2E_REGION - New Relic region: "staging" or "production" (default: production)
1011
# NR_ACCOUNT_ID - New Relic account ID
1112
# NR_LICENSE_KEY - New Relic license key (used as the Events API ingest key)
1213
#
@@ -17,7 +18,16 @@ set -euo pipefail
1718

1819
END_TIME=$(date +%s)
1920
DURATION=$((END_TIME - JOB_START_TIME))
20-
ACCOUNT="Agent Control Canaries (prod)"
21+
22+
# Set insights collector URL based on region
23+
E2E_REGION="${E2E_REGION:-production}"
24+
if [[ "$E2E_REGION" == "staging" ]]; then
25+
INSIGHTS_COLLECTOR_URL="https://staging-insights-collector.newrelic.com"
26+
ACCOUNT="Agent Control Canaries (staging)"
27+
else
28+
INSIGHTS_COLLECTOR_URL="https://insights-collector.newrelic.com"
29+
ACCOUNT="Agent Control Canaries (prod)"
30+
fi
2131

2232
# GITHUB_HEAD_REF is set for pull requests; GITHUB_REF_NAME covers push/schedule/dispatch.
2333
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-}}"
@@ -48,7 +58,7 @@ events=$(jq -n \
4858
}]')
4959

5060
curl -s -X POST \
51-
"https://insights-collector.newrelic.com/v1/accounts/${NR_ACCOUNT_ID}/events" \
61+
"${INSIGHTS_COLLECTOR_URL}/v1/accounts/${NR_ACCOUNT_ID}/events" \
5262
-H "Content-Type: application/json" \
5363
-H "Api-Key: ${NR_LICENSE_KEY}" \
5464
-d "$events"

‎Tiltfile‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,23 @@ sa_chart_values_file = os.getenv('SA_CHART_VALUES_FILE','local/agent-control-til
1313
cluster = os.getenv('CLUSTER', "")
1414
# Branch of the helm-charts repo to use.
1515
feature_branch = os.getenv('FEATURE_BRANCH', "master")
16+
region = os.getenv('REGION', "US")
17+
18+
# Region-specific fleet IDs and organization IDs for k8s e2e tests
19+
# These values are used to override the hardcoded values in the ac-values-*.yml files
20+
if region.lower() == "staging":
21+
# Staging account fleet IDs
22+
# For now using production IDs as placeholders
23+
fleet_id_proxy = "MTIyMTMwNjh8TkdFUHxGTEVFVHwwMTlkMjVhNy0zY2ViLTdhMjUtODZhOS0yYzhiMzQwNDdhMDk" # ac-e2e-empty-deployment
24+
fleet_id_fleet_control = "MTIyMTMwNjh8TkdFUHxGTEVFVHwwMTlkMjVhOC04NTI2LTdkZDItOWVkMC1mYjM5YmMxYjdlZTE" # ac-e2e-3
25+
fleet_id_apm = "MTIyMTMwNjh8TkdFUHxGTEVFVHwwMTlkMjVhZS02MjQ4LTc1NmYtOGE0Yi1kNWE3OWZmNGU1YTA" # ac-e2e-apm
26+
organization_id = "9d789cca-f661-458d-be06-882d1e6e409d"
27+
else:
28+
# Production account fleet IDs (Agent Control Canaries)
29+
fleet_id_proxy = "NjQyNTg2NXxOR0VQfEZMRUVUfDAxOTgwOTZiLTk2NzctNzllNi05OWJmLTQ4YWRlOTRiYmZjMw" # ac-e2e-empty-deployment
30+
fleet_id_fleet_control = "NjQyNTg2NXxOR0VQfEZMRUVUfDAxOTkyODk4LTg0OWMtNzdmZC1iN2Y1LWMwYjNiNDRmNzlkNw" # ac-e2e-3
31+
fleet_id_apm = "NjQyNTg2NXxOR0VQfEZMRUVUfDAxOWI0NTkxLTYyYmEtN2RiZC04OGY5LWQ1MGMzZDU2YTM3YQ" # ac-e2e-apm
32+
organization_id = "b961cf81-d62b-4359-8822-7b1d6dadd374"
1633

1734
# Remote updates are disabled by default in order to avoid accidental downgrades.
1835
enable_ac_remote_update = os.getenv('ENABLE_AC_REMOTE_UPDATE', "false")
@@ -118,8 +135,33 @@ ac_flags = [
118135
'--version=>=0.0.0-beta',
119136
'--set=agentControlDeployment.chartValues.image.imagePullPolicy=Always',
120137
'--values=' + sa_chart_values_file,
138+
# Override fleet IDs and organization ID based on region
139+
'--set=agentControlDeployment.chartValues.systemIdentity.organizationId=' + organization_id,
121140
]
122141

142+
# Add region-specific fleet IDs based on the test scenario
143+
# These will be used by tests that have fleet_control enabled
144+
if 'proxy' in sa_chart_values_file:
145+
ac_flags.append('--set=agentControlDeployment.chartValues.config.fleet_control.fleet_id=' + fleet_id_proxy)
146+
elif 'fleet-control' in sa_chart_values_file:
147+
ac_flags.append('--set=agentControlDeployment.chartValues.config.fleet_control.fleet_id=' + fleet_id_fleet_control)
148+
elif 'apm' in sa_chart_values_file:
149+
ac_flags.append('--set=agentControlDeployment.chartValues.config.fleet_control.fleet_id=' + fleet_id_apm)
150+
151+
# Set nrStaging based on region
152+
if region.lower() == "staging":
153+
ac_flags.append('--set=agentControlDeployment.chartValues.nrStaging=true')
154+
# Also set nrStaging for specific agent configs that need it
155+
if 'proxy' in sa_chart_values_file:
156+
ac_flags.append('--set=agentControlDeployment.chartValues.agentsConfig.prometheus.nrStaging=true')
157+
elif 'fleet-control' in sa_chart_values_file:
158+
ac_flags.append('--set=agentControlDeployment.chartValues.agentsConfig.infra.chart_values.nrStaging=true')
159+
elif 'custom-repo' in sa_chart_values_file:
160+
# custom-repo uses nri-bundle chart with nested prometheus agent config
161+
ac_flags.append('--set=agentControlDeployment.chartValues.agentsConfig.prometheus.chart_values.newrelic-prometheus-agent.newrelic-prometheus-agent.nrStaging=true')
162+
else:
163+
ac_flags.append('--set=agentControlDeployment.chartValues.nrStaging=false')
164+
123165
ac_chart_deps = ['build-binary', 'local-child-chart-upload']
124166

125167
latest_flux = os.getenv('USE_LATEST_FLUX', 'false').lower() == 'true'

‎agent-control/src/cli/common/region.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const PUBLIC_KEY_ENDPOINT_EU: &str =
1616
const PUBLIC_KEY_ENDPOINT_STAGING: &str =
1717
"https://staging-publickeys.newrelic.com/r/blob-management/global/agentconfiguration/jwks.json";
1818

19-
const OTLP_URL_STAGING: &str = "staging.otlp.nr-data.net";
19+
const OTLP_URL_STAGING: &str = "staging-otlp.nr-data.net";
2020
const OTLP_URL_EU: &str = "otlp.eu01.nr-data.net";
2121
const OTLP_URL_US: &str = "otlp.nr-data.net";
2222

@@ -161,7 +161,7 @@ mod tests {
161161
#[rstest]
162162
#[case(Region::US, "https://otlp.nr-data.net:4317/")]
163163
#[case(Region::EU, "https://otlp.eu01.nr-data.net:4317/")]
164-
#[case(Region::STAGING, "https://staging.otlp.nr-data.net:4317/")]
164+
#[case(Region::STAGING, "https://staging-otlp.nr-data.net:4317/")]
165165
fn test_otel_endpoint(#[case] region: Region, #[case] expected_endpoint: &str) {
166166
assert_eq!(
167167
region.otel_endpoint().to_string(),

‎test/e2e-runner/src/common.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ pub mod file;
66
pub mod logs;
77
pub mod nrql;
88
pub mod on_drop;
9+
pub mod region_data;
910
pub mod test;
1011

1112
/// Arguments to be set for every test that needs Agent Control installation

‎test/e2e-runner/src/common/region_data.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/// Returns the appropriate fleet ID based on the region.
2+
/// Returns the staging fleet ID if region is "staging", otherwise returns the prod fleet ID.
3+
pub fn get_fleet_id(region: &str, prod_fleet_id: &str, staging_fleet_id: &str) -> String {
4+
match region.to_lowercase().as_str() {
5+
"staging" => staging_fleet_id.to_string(),
6+
_ => prod_fleet_id.to_string(),
7+
}
8+
}

‎test/e2e-runner/src/linux/scenarios/ebpf_agent.rs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ pub fn test_ebpf_agent(args: Args) {
2020
.clone()
2121
.expect("--infra-agent-version is required for this scenario");
2222

23+
let staging = matches!(args.nr_region.to_lowercase().as_str(), "staging");
24+
2325
let recipe_data = RecipeData {
2426
args,
2527
monitoring_source: "infra-agent".to_string(),
@@ -50,22 +52,31 @@ agents:
5052
);
5153
config::update_config(linux::DEFAULT_AC_CONFIG_PATH, config);
5254
// eBPF agent config
53-
write_agent_local_config(
54-
&linux::local_config_path("nr-ebpf"),
55+
let ebpf_config = if staging {
5556
format!(
5657
r#"
5758
config_agent:
5859
DEPLOYMENT_NAME: {test_id}
60+
OTLP_ENDPOINT: staging-otlp.nr-data.net:443
5961
"#
60-
),
61-
);
62+
)
63+
} else {
64+
format!(
65+
r#"
66+
config_agent:
67+
DEPLOYMENT_NAME: {test_id}
68+
"#
69+
)
70+
};
71+
write_agent_local_config(&linux::local_config_path("nr-ebpf"), ebpf_config);
6272
// Infra agent config: it is used to generate traffic for eBPF metrics to appear
6373
write_agent_local_config(
6474
&linux::local_config_path("nr-infra"),
6575
format!(
6676
r#"
6777
config_agent:
6878
license_key: '{{{{NEW_RELIC_LICENSE_KEY}}}}'
79+
staging: {staging}
6980
version: {}
7081
"#,
7182
infra_version

‎test/e2e-runner/src/linux/scenarios/infra_agent.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ pub fn test_installation_with_infra_agent(args: Args) {
1818
.clone()
1919
.expect("--infra-agent-version is required for this scenario");
2020

21+
let staging = matches!(args.nr_region.to_lowercase().as_str(), "staging");
22+
2123
let recipe_data = RecipeData {
2224
args,
2325
monitoring_source: "infra-agent".to_string(),
@@ -55,8 +57,7 @@ agents:
5557
r#"
5658
config_agent:
5759
license_key: '{{{{NEW_RELIC_LICENSE_KEY}}}}'
58-
log:
59-
level: debug
60+
staging: {staging}
6061
config_logging:
6162
logging.yml:
6263
logs:

0 commit comments

Comments
 (0)