Skip to content

Commit d007978

Browse files
author
Satheesh Rajendran
committed
Add support for profilers
Let's add support for profilers to framework, this would help us capture the snapshot of system details through any user provided commands running in a predefined frequent intervals and store in a file in test-reports which then can be used for further processing later. Usage: `profilers` file in the basepath documents how user can create one profiler instance and running the test with `--enable-profiler` will allow the framework to enable the profiler threads run in parallel to the test and collect the output and profiler threads gets stopped at the end of tests. Signed-off-by: Satheesh Rajendran <[email protected]>
1 parent df857a3 commit d007978

File tree

4 files changed

+253
-0
lines changed

4 files changed

+253
-0
lines changed

‎OpTestConfiguration.py

+6
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,12 @@ def get_parser():
413413
hmcgroup.add_argument(
414414
"--lpar-vios", help="Lpar VIOS to boot before other LPARS", default=None)
415415

416+
profilergroup = parser.add_argument_group('Profiler',
417+
'Profiler enable commands')
418+
profilergroup.add_argument("--enable-profiler", help="If set, profilers will be enabled",
419+
action='store_true', default=False)
420+
profilergroup.add_argument("--profiler-file", help="provide the profilers file, profilers given in the file will be enabled",
421+
default="./profilers")
416422
return parser
417423

418424

‎common/OpTestProfiler.py

+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
#!/usr/bin/env python3
2+
# OpenPOWER Automated Test Project
3+
#
4+
# Contributors Listed Below - COPYRIGHT 2019
5+
# [+] International Business Machines Corp.
6+
#
7+
#
8+
# Licensed under the Apache License, Version 2.0 (the "License");
9+
# you may not use this file except in compliance with the License.
10+
# You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
17+
# implied. See the License for the specific language governing
18+
# permissions and limitations under the License.
19+
#
20+
21+
'''
22+
Profiler library
23+
---------------------
24+
This adds a support to add user defined profilers
25+
'''
26+
27+
import os
28+
import time
29+
import threading
30+
31+
import OpTestConfiguration
32+
from .OpTestSystem import OpSystemState
33+
from .Exceptions import CommandFailed
34+
35+
import OpTestLogger
36+
log = OpTestLogger.optest_logger_glob.get_logger(__name__)
37+
38+
39+
class profilerThread(threading.Thread):
40+
def __init__(self, cmd):
41+
threading.Thread.__init__(self)
42+
self.env = cmd['env']
43+
self.cmd = cmd['cmd']
44+
self.freq = int(cmd['freq'])
45+
self.name = cmd['name'] if cmd['name'] else self.cmd.replace(' ', '_')
46+
self._stop_event = threading.Event()
47+
self.conf = OpTestConfiguration.conf
48+
# TODO: consider adding all profiler output into seperate folder
49+
self.host = self.conf.host()
50+
self.system = self.conf.system()
51+
self.console = None
52+
if self.env == 'sut':
53+
try:
54+
self.console = self.host.get_new_ssh_connection(self.name)
55+
except Exception as err:
56+
# might not be yet in OS state
57+
pass
58+
elif self.env == 'server':
59+
pass
60+
elif self.env == 'bmc':
61+
pass
62+
else:
63+
log.warning("Unknown env given to run profilers, give either sut to"
64+
"run inside host or server to run ipmi commands")
65+
66+
def run(self):
67+
log.info("Starting profile %s" % self.name)
68+
self.executed = False
69+
while True:
70+
if self.freq > 0:
71+
if self.env == 'sut':
72+
if self.system.state != OpSystemState.OS:
73+
continue
74+
if self.console:
75+
try:
76+
output = self.console.run_command(self.cmd)
77+
except CommandFailed as cf:
78+
log.warning('Profiler cmd failed to run %s', self.cmd)
79+
else:
80+
# try to reconnect
81+
log.warning('Reconnecting SSH console...')
82+
self.console = self.host.get_new_ssh_connection(self.name)
83+
84+
elif self.env == 'server':
85+
# TODO:
86+
log.warning("Yet to implement")
87+
break
88+
elif self.env == 'bmc':
89+
# TODO:
90+
log.warning("Yet to implement")
91+
break
92+
time.sleep(self.freq)
93+
if self.is_stopped():
94+
break
95+
96+
else:
97+
if not self.executed:
98+
# FIXME: NEED add support for running long run cmds
99+
if self.env == 'sut':
100+
if self.system.state != OpSystemState.OS:
101+
continue
102+
if self.console:
103+
try:
104+
output = self.console.run_command(self.cmd)
105+
except CommandFailed as cf:
106+
log.warning('Profiler cmd failed to run %s', self.cmd)
107+
else:
108+
self.console = self.host.get_new_ssh_connection(self.name)
109+
try:
110+
output = self.console.run_command(self.cmd)
111+
except CommandFailed as cf:
112+
log.warning('Profiler cmd failed to run %s', self.cmd)
113+
elif self.env == 'server':
114+
# TODO:
115+
log.warning("Yet to implement")
116+
break
117+
elif self.env == 'bmc':
118+
# TODO:
119+
log.warning("Yet to implement")
120+
break
121+
self.executed = True
122+
if self.is_stopped():
123+
break
124+
125+
def stop(self):
126+
log.info("Stopping profile %s", self.name)
127+
self._stop_event.set()
128+
129+
def is_stopped(self):
130+
return self._stop_event.is_set()
131+
132+
def wait(self, delaysec=5):
133+
self._stop_event.wait(delaysec)
134+
135+
136+
class Profilers(object):
137+
def __init__(self, profiler_cmd_path=None, profiler_cmd=None):
138+
"""
139+
Profiler class to create profiler threads
140+
params: profiler_cmd_path: file with profiler information,by default it
141+
will use the 'profilers' file kept in basepath
142+
params: profiler_cmd: dict type optional profile, if given will take the
143+
precedence over profiler_cmd_path argument,
144+
can be used inside testcase, E:g:-
145+
{'cmd': vmstat,
146+
'freq': 2,
147+
'env': 'sut',
148+
'name': 'vmstat-1'}
149+
"""
150+
self.conf = OpTestConfiguration.conf
151+
self.path = profiler_cmd_path if profiler_cmd_path else os.join(os.path.dirname(os.path.abspath(__file__)), 'profilers')
152+
if not os.path.isfile(self.path):
153+
log.warning("Check the profiler command path, given path is not valid: %s", self.path)
154+
self.profilers = []
155+
# Optional and if given takes precedence
156+
if profiler_cmd:
157+
self.profilers.append(profiler_cmd)
158+
else:
159+
self.profilers = self.parse_profilers()
160+
self.host = self.conf.host()
161+
self.system = self.conf.system()
162+
self.profilethreads = []
163+
164+
def parse_profilers(self):
165+
profile_content = []
166+
profile_list = []
167+
profile = {'cmd': None,
168+
'freq': 0,
169+
'env': 'sut',
170+
'name': None}
171+
temp = profile.copy()
172+
try:
173+
with open(self.path) as profile_obj:
174+
profile_content = [line.strip('\n') for line in profile_obj.readlines()]
175+
except Exception as err:
176+
log.warning("Error reading profiler cmd file")
177+
pass
178+
else:
179+
for item in profile_content:
180+
if item.startswith("#"):
181+
continue
182+
try:
183+
temp['cmd'] = item.split(',')[0]
184+
temp['freq'] = int(item.split(',')[1])
185+
temp['env'] = item.split(',')[2]
186+
temp['name'] = item.split(',')[3]
187+
except IndexError:
188+
pass
189+
profile_list.append(temp.copy())
190+
temp = profile.copy()
191+
finally:
192+
return profile_list
193+
194+
def create_profile_threads(self):
195+
profiler_threads = []
196+
for prof in self.profilers:
197+
self.profilethreads.append(profilerThread(prof))
198+
return self.profilethreads
199+
200+
def run(self):
201+
self.create_profile_threads()
202+
for thread in self.profilethreads:
203+
thread.start()
204+
205+
def stop(self):
206+
for thread in self.profilethreads:
207+
thread.stop()
208+
209+
def join(self):
210+
for thread in self.profilethreads:
211+
thread.join()

‎op-test

+8
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ from testcases import OpTestSensors
102102
from testcases import OpTestSwitchEndianSyscall
103103
from testcases import OpTestHostboot
104104
from testcases import OpTestExample
105+
from common.OpTestProfiler import Profilers
105106
import OpTestConfiguration
106107
import sys
107108
import time
@@ -934,6 +935,10 @@ try:
934935
OpTestConfiguration.conf.util.cleanup()
935936
sys.exit(exit_code)
936937

938+
# create profiler instances
939+
if OpTestConfiguration.conf.args.enable_profiler:
940+
profiler = Profilers(profiler_cmd_path=OpTestConfiguration.conf.args.profiler_file)
941+
profiler.run()
937942
if not res or (res and not (res.errors or res.failures)):
938943
res = run_tests(t, failfast=OpTestConfiguration.conf.args.failfast)
939944
else:
@@ -968,6 +973,9 @@ except Exception as e:
968973
exit_code = -1
969974
sys.exit(exit_code)
970975
finally:
976+
# stop profile instances
977+
if OpTestConfiguration.conf.args.enable_profiler:
978+
profiler.stop()
971979
# Create a softlink to `latest` test results
972980
output = OpTestConfiguration.conf.logdir
973981
if not os.path.exists(output):

‎profilers

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Supported format to run profile
2+
# command,freqency in seconds,where to run,name of profiler
3+
# Eg:-
4+
# date,2,sut,date-1
5+
#
6+
# Above line will create a profiler which runs `date`
7+
# command every 2 seconds inside host using SSH session
8+
# and stores in a file which named *date-1*.log
9+
# in respective test-reports folder.
10+
#
11+
# command: Any command that is available in the place where it runs.
12+
#
13+
# freqency in seconds: Takes any integer value, `0` is a special value
14+
# where given command itself will run in batch and no
15+
# need to run the command in intervals.
16+
#
17+
# where to run: Currently supports only in `sut` ie. Host for which
18+
# test is run.
19+
# TODO:-
20+
# server - runs commands in the server where optest runs.
21+
# bmc - runs commands inside bmc.
22+
#
23+
# name of profiler: Name to be used to represent the profiler
24+
# bydefault, command name is used.
25+
#
26+
#date,2,sut,date-1
27+
#vmstat 1,0,sut,test1
28+
#date,2,sut

0 commit comments

Comments
 (0)