Skip to content

Commit e44a06d

Browse files
tannalynntannalynn
authored
Merge pull request #3302 from newrelic/add_perfverse
Add Perfverse
2 parents cb33960 + be7994a commit e44a06d

File tree

90 files changed

+2286
-3
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+2286
-3
lines changed

‎.github/actions/run_perfverse/action.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: 'Run Perfverse'
2+
description: 'Runs performance testing on the specified agent tag'
3+
inputs:
4+
agent_tag:
5+
description: 'What agent version to install'
6+
required: true
7+
agent_env_vars:
8+
description: ''
9+
required: false
10+
run_time:
11+
description: ''
12+
required: true
13+
nr_license_key:
14+
description: ''
15+
required: true
16+
test_tag:
17+
description: ''
18+
required: true
19+
docker_monitor_output_dir:
20+
description: ''
21+
required: true
22+
default: docker_monitor_output
23+
run_label:
24+
description: ''
25+
required: true
26+
iterations:
27+
description: ''
28+
required: true
29+
default: 3
30+
31+
runs:
32+
using: "composite"
33+
steps:
34+
- name: Install Ruby
35+
uses: ruby/setup-ruby@44511735964dcb71245e7e55f72539531f7bc0eb # tag v1.257.0
36+
with:
37+
ruby-version: 3.4
38+
39+
- name: Run ruby script
40+
shell: bash
41+
run: |
42+
ruby ./.github/workflows/scripts/run_perf_tests.rb
43+
env:
44+
AGENT_TAG: ${{ inputs.agent_tag }}
45+
TEST_TAG: ${{ inputs.test_tag }}
46+
RUN_TIME: ${{ inputs.run_time }}
47+
NR_LICENSE_KEY: ${{ inputs.nr_license_key }}
48+
DOCKER_MONITOR_OUTPUT_DIR: ${{ inputs.docker_monitor_output_dir }}
49+
ITERATIONS: ${{ inputs.iterations }}
50+
51+
- name: Upload docker report
52+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # tag v4.6.2
53+
with:
54+
if-no-files-found: error
55+
path: test/perfverse/docker_monitor/${{ inputs.docker_monitor_output_dir}}/
56+
name: docker_monitor_report-${{ inputs.run_label }}
57+
retention-days: 1
58+
59+
60+
61+
62+
63+
64+
65+
66+
67+
68+
69+

‎.github/workflows/run_perfverse.yml‎

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,39 @@ permissions:
33
contents: read
44
on:
55
workflow_dispatch:
6+
inputs:
7+
run_1:
8+
description: "Git tag and env vars for runs. Format: \"git_tag:ENV_VAR_1=one;ENV_VAR_2=two\"\n 1st run"
9+
required: true
10+
type: string
11+
run_2:
12+
description: "2nd run"
13+
required: false
14+
type: string
15+
run_3:
16+
description: "3rd run"
17+
required: false
18+
type: string
19+
run_4:
20+
description: "4th run"
21+
required: false
22+
type: string
23+
agent_test_agent_disabled:
24+
description: "Test run with agent disabled?"
25+
default: true
26+
required: true
27+
type: boolean
28+
run_time:
29+
description: "Run time"
30+
default: 5m
31+
required: true
32+
type: string
33+
iterations:
34+
description: "Number of iterations to run"
35+
default: 5
36+
required: true
37+
type: string
38+
# maybe add a name for each run so its better on graph
639

740

841

@@ -14,11 +47,118 @@ jobs:
1447
run: 'git config --global init.defaultBranch main'
1548
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
1649

17-
- run: echo 'hi im not doin nothin'
50+
- name: run 1st version
51+
uses: ./.github/actions/run_perfverse
52+
with:
53+
agent_tag: ${{ inputs.run_1 }}
54+
run_time: ${{ inputs.run_time }}
55+
nr_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
56+
iterations: ${{ inputs.iterations }}
57+
run_label: run_one
1858

59+
run_perf_tests_2:
60+
runs-on: ubuntu-latest
61+
if: ${{ inputs.run_2 != '' }}
62+
steps:
63+
- name: Configure git
64+
run: 'git config --global init.defaultBranch main'
65+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
1966

67+
- name: run 2nd version
68+
uses: ./.github/actions/run_perfverse
69+
with:
70+
agent_tag: ${{ inputs.run_2 }}
71+
run_time: ${{ inputs.run_time }}
72+
nr_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
73+
iterations: ${{ inputs.iterations }}
74+
run_label: run_two
75+
76+
run_perf_tests_3:
77+
runs-on: ubuntu-latest
78+
if: ${{ inputs.run_3 != '' }}
79+
steps:
80+
- name: Configure git
81+
run: 'git config --global init.defaultBranch main'
82+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
83+
84+
- name: run 3rd version
85+
uses: ./.github/actions/run_perfverse
86+
with:
87+
agent_tag: ${{ inputs.run_3 }}
88+
run_time: ${{ inputs.run_time }}
89+
nr_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
90+
iterations: ${{ inputs.iterations }}
91+
run_label: run_three
92+
93+
run_perf_tests_4:
94+
runs-on: ubuntu-latest
95+
if: ${{ inputs.run_4 != '' }}
96+
steps:
97+
- name: Configure git
98+
run: 'git config --global init.defaultBranch main'
99+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
100+
101+
- name: run 4th version
102+
uses: ./.github/actions/run_perfverse
103+
with:
104+
agent_tag: ${{ inputs.run_4 }}
105+
run_time: ${{ inputs.run_time }}
106+
nr_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
107+
iterations: ${{ inputs.iterations }}
108+
run_label: run_four
109+
110+
111+
run_perf_tests_disabled:
112+
runs-on: ubuntu-latest
113+
if: ${{ inputs.agent_test_agent_disabled }}
114+
steps:
115+
- name: Configure git
116+
run: 'git config --global init.defaultBranch main'
117+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
118+
119+
- name: run agent disabled version
120+
uses: ./.github/actions/run_perfverse
121+
with:
122+
agent_tag: "AGENT_DISABLED:NEW_RELIC_AGENT_ENABLED=false"
123+
run_time: ${{ inputs.run_time }}
124+
nr_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
125+
iterations: ${{ inputs.iterations }}
126+
run_label: run_agent_disabled
127+
128+
create_graphs:
129+
needs: [run_perf_tests_disabled, run_perf_tests_4, run_perf_tests_3, run_perf_tests_2, run_perf_tests_1]
130+
if: always()
131+
runs-on: ubuntu-latest
132+
steps:
133+
- name: Configure git
134+
run: 'git config --global init.defaultBranch main'
135+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag v5.0.0
20136

137+
- name: create dirs
138+
run: |
139+
mkdir output
140+
mkdir inputs
141+
working-directory: ./test/perfverse/reports
21142

143+
- name: Download all workflow run artifacts
144+
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # tag v5.0.0
145+
with:
146+
path: test/perfverse/reports/inputs
22147

148+
- name: Build docker image to make graphs
149+
run: |
150+
docker build --pull --progress=plain -t charty:local .
151+
working-directory: ./test/perfverse/reports
23152

153+
- name: Run docker image to make graphs
154+
run: |
155+
docker run --rm --name ruby-charty --mount type=bind,source=./output,target=/charty/output charty:local
156+
working-directory: ./test/perfverse/reports
24157

158+
- name: Upload graphs
159+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # tag v4.6.2
160+
with:
161+
if-no-files-found: error
162+
path: test/perfverse/reports/output/
163+
name: graphs
164+
retention-days: 14

‎.github/workflows/scripts/run_perf_tests.rb‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# frozen_string_literal: true
2+
3+
def output_line(str)
4+
puts "*" * 120
5+
puts str
6+
end
7+
8+
def run_command(command)
9+
puts "Running command: #{command}"
10+
`#{command}`
11+
end
12+
13+
def transform_agent_tags(agent_tag)
14+
agent_tag.split(':').tap do |array|
15+
array[1] = array[1]&.split(';')
16+
end
17+
end
18+
19+
def build_rails_app(git_tag)
20+
output_line("Building rails app with git tag #{git_tag}")
21+
run_command("cd ./test/perfverse/ && docker build --pull --build-arg AGENT_VERSION=#{git_tag} --progress=plain -t ruby_perf_app:local .")
22+
end
23+
24+
def build_docker_monitor_report
25+
output_line("Building docker monitor image")
26+
run_command('cd ./test/perfverse/docker_monitor && docker build --pull --progress=plain -t docker_monitor_report:local . ')
27+
end
28+
29+
def pull_locust
30+
output_line("Pulling locust docker image")
31+
run_command("docker pull locustio/locust")
32+
end
33+
34+
def shutdown_rails_app(container_id)
35+
output_line("Shutting down rails app")
36+
run_command("docker stop #{container_id}")
37+
end
38+
39+
def run_traffic
40+
output_line("Running locust traffic with #{ENV['RUN_TIME']} duration")
41+
run_command("cd ./test/perfverse/traffic && docker run -p 8089:8089 --network=\"host\" -v $PWD:/mnt/locust locustio/locust -t $RUN_TIME -f /mnt/locust/driver.py --host=http://127.0.0.1:3000 --headless -u 5")
42+
end
43+
44+
def run_rails_app(agent_tag, env_vars, iteration)
45+
env_str = ''
46+
env_vars&.each do |env_var|
47+
env_str += "-e #{env_var} "
48+
end
49+
50+
app_name = "ruby_perf_app_#{ENV['TEST_TAG']}_#{agent_tag}_#{iteration}"
51+
output_line("Running ruby app in background. Name: #{app_name}")
52+
cpu_mem = '--cpus 4 --memory 2G'
53+
54+
Thread.new do
55+
run_command("cd ./test/perfverse/ && docker run --rm --name #{app_name} #{cpu_mem} #{env_str} -e NEW_RELIC_LICENSE_KEY=$NR_LICENSE_KEY -e NEW_RELIC_APP_NAME=#{app_name} -e NEW_RELIC_HOST=staging-collector.newrelic.com -e s -p 3000:3000 ruby_perf_app:local")
56+
end
57+
sleep 2
58+
thread = run_docker_report(agent_tag, app_name, iteration)
59+
sleep 1
60+
61+
[app_name, thread]
62+
end
63+
64+
def run_docker_report(agent_tag, container_ids, iteration)
65+
Thread.new do
66+
output_dir = "#{ENV['DOCKER_MONITOR_OUTPUT_DIR']}/run_#{iteration}"
67+
env_str = ''
68+
env_str += "-e TEST_TAG=#{ENV['TEST_TAG']} "
69+
env_str += "-e AGENT_VERSION=#{agent_tag} "
70+
env_str += "-e DOCKER_MONITOR_OUTPUT_DIR=#{output_dir} "
71+
env_str += "-e MONITOR_CONTAINERS=#{container_ids} "
72+
73+
docker_mount_bind = "--mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock"
74+
output_mount_bind = "--mount type=bind,source=./#{output_dir},target=/app/#{output_dir}"
75+
76+
command = "cd ./test/perfverse/docker_monitor && mkdir -p #{output_dir} && "
77+
command << "sudo docker run --rm --name docker_monitor_report #{env_str} #{docker_mount_bind} #{output_mount_bind} docker_monitor_report:local"
78+
79+
output_line("Running docker monitor report")
80+
output = run_command(command)
81+
82+
output_line("Docker Monitor Report Output: \n" + output)
83+
end
84+
end
85+
86+
###############################################################################
87+
88+
iterations = ENV['ITERATIONS'].to_i
89+
agent_tag, env_vars = transform_agent_tags(ENV['AGENT_TAG'])
90+
output_line("Running perf test #{iterations} times for #{ENV['RUN_TIME']} with agent tag #{agent_tag} and env vars #{env_vars}")
91+
92+
pull_locust
93+
build_docker_monitor_report
94+
95+
iterations.times do |i|
96+
build_rails_app(agent_tag)
97+
98+
app_name, monitor_thread = run_rails_app(agent_tag, env_vars, i)
99+
run_traffic
100+
101+
shutdown_rails_app(app_name)
102+
monitor_thread.join
103+
end
104+

‎.github/workflows/scripts/setup_bundler‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ function using_old_ruby {
2626
function update_to_desired_rubygems_version {
2727
# Older rubies come with older Rubygems and we need 3.0.6 to
2828
# correctly install Bundler 1.17 for the multiverse test suite
29-
# Rubies < 2.3 need to use update_rubygems,
30-
# newer Rubies can use 'gem update --system'
29+
3130
if [[ $RUBY_VERSION =~ ^2\.[^7] ]]; then
3231
echo "DEBUG: running 'gem update --system 3.0.6 --force'"
3332
gem update --system 3.0.6 --force >/dev/null

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,6 @@ Brewfile.lock.json
4444
test/minitest/minitest_time_report
4545
gem_manifest_*.json
4646
yarn-error.log
47+
48+
test/perfverse/reports/inputs
49+
test/perfverse/reports/output

‎.rubocop.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ AllCops:
1414
Exclude:
1515
- 'test/multiverse/suites/active_record/db/schema.rb'
1616
- 'test/multiverse/suites/active_record_pg/db/schema.rb'
17+
- 'test/perfverse/rails7/**/*'
1718
NewCops: enable
1819

1920
Bundler/DuplicatedGem:

‎test/perfverse/Dockerfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
ARG RUBY_VERSION=3.3.3
2+
FROM registry.docker.com/library/ruby:$RUBY_VERSION as base
3+
4+
ARG AGENT_VERSION
5+
6+
# Fail the build if AGENT_VERSION is not provided
7+
RUN test -n "$AGENT_VERSION"
8+
9+
10+
WORKDIR /usr/src/app
11+
COPY rails7 .
12+
COPY newrelic.yml ./config/
13+
COPY --chmod=0755 bin/set-agent-version.sh /set-agent-version.sh
14+
COPY --chmod=0755 entrypoint.sh /entrypoint.sh
15+
16+
# Set agent version
17+
RUN /set-agent-version.sh
18+
19+
# RUN gem install date
20+
RUN bundle install \
21+
&& cat /usr/src/app/Gemfile.lock | grep newrelic_rpm
22+
RUN bundle exec rake db:setup
23+
RUN bundle exec rake assets:precompile
24+
25+
ENTRYPOINT ["/entrypoint.sh"]

0 commit comments

Comments
 (0)