Skip to content

Commit 5ff973b

Browse files
author
zhangjipeng
committed
fix: adjust perf_test logic
Signed-off-by: zhangjipeng <zhangjipeng@xiaomi.com>
1 parent 53622e8 commit 5ff973b

2 files changed

Lines changed: 30 additions & 12 deletions

File tree

perf_tests/test.cpp

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include <sched.h>
4141
#include <sys/resource.h>
4242
#include <unistd.h>
43+
#include <errno.h>
4344
#endif
4445

4546
static uint8_t* test_buffer = NULL;
@@ -191,7 +192,12 @@ static void set_process_priority(void)
191192
SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS);
192193
SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST);
193194
#else
194-
setpriority(PRIO_PROCESS, 0, -20);
195+
errno = 0;
196+
if (setpriority(PRIO_PROCESS, 0, -20) != 0 && errno != 0) {
197+
std::cerr << "[Perf Warning] setpriority(-20) failed (errno=" << errno
198+
<< "); results may be noisy. Run as root or with CAP_SYS_NICE."
199+
<< std::endl;
200+
}
195201
#endif
196202
}
197203

@@ -202,10 +208,18 @@ static void set_cpu_affinity(void)
202208
SetProcessAffinityMask(GetCurrentProcess(), mask);
203209
SetThreadAffinityMask(GetCurrentThread(), mask);
204210
#else
211+
// Avoid CPU 0: on Linux it usually handles the most interrupts and is the
212+
// noisiest core. Pin to the last online CPU instead.
213+
long ncpu = sysconf(_SC_NPROCESSORS_ONLN);
214+
int target = (ncpu > 1) ? (int)(ncpu - 1) : 0;
205215
cpu_set_t cpuset;
206216
CPU_ZERO(&cpuset);
207-
CPU_SET(0, &cpuset);
208-
sched_setaffinity(0, sizeof(cpuset), &cpuset);
217+
CPU_SET(target, &cpuset);
218+
if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) {
219+
std::cerr << "[Perf Warning] sched_setaffinity(cpu=" << target
220+
<< ") failed (errno=" << errno
221+
<< "); results may be noisy due to core migration." << std::endl;
222+
}
209223
#endif
210224
}
211225

perf_tests/test.h

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,9 @@ class PerformanceTest : public ::testing::Test
238238
);
239239

240240
BenchmarkResult result;
241-
result.avg_ms = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
241+
// Use trimmed data for avg as well, to keep the same statistical basis
242+
// as std_dev/min/max/mid which are all computed on filtered_times.
243+
result.avg_ms = std::accumulate(filtered_times.begin(), filtered_times.end(), 0.0) / filtered_times.size();
242244
result.min_ms = filtered_times.front();
243245
result.max_ms = filtered_times.back();
244246
result.iterations = run_count;
@@ -335,7 +337,9 @@ class PerformanceTest : public ::testing::Test
335337
<< ", total: " << (result.total_time_ms / 1000.0) << "s)" << std::endl;
336338
} else {
337339
const auto& baseline = baseline_data[key];
338-
double diff_percent = ((result.mid_ms - baseline.mid_ms) / baseline.mid_ms) * 100.0;
340+
// Use the trimmed mean (avg_ms) consistently for diff, direction and
341+
// the t-test below, so the whole decision path shares one statistic.
342+
double diff_percent = ((result.avg_ms - baseline.avg_ms) / baseline.avg_ms) * 100.0;
339343

340344
// Statistical comparison
341345
double t_stat, p_value;
@@ -348,23 +352,23 @@ class PerformanceTest : public ::testing::Test
348352
// No statistically significant difference
349353
std::cout << "[No Significant Change] " << test_name << ": "
350354
<< std::setprecision(6)
351-
<< "median: " << result.mid_ms << " ms "
352-
<< "(baseline: " << baseline.mid_ms << " ms, "
355+
<< "avg: " << result.avg_ms << " ms "
356+
<< "(baseline: " << baseline.avg_ms << " ms, "
353357
<< "diff: " << diff_percent << "%, "
354358
<< "p=" << p_value << ", not significant)" << std::endl;
355359
} else if (!exceeds_threshold) {
356360
// Statistically significant but below threshold - acceptable
357361
std::cout << "[Acceptable Change] " << test_name << ": "
358362
<< std::setprecision(6)
359-
<< "median: " << result.mid_ms << " ms "
360-
<< "(baseline: " << baseline.mid_ms << " ms, "
363+
<< "avg: " << result.avg_ms << " ms "
364+
<< "(baseline: " << baseline.avg_ms << " ms, "
361365
<< "diff: " << diff_percent << "%, "
362366
<< "below " << PERF_REGRESSION_THRESHOLD << "% threshold)" << std::endl;
363-
} else if (result.mid_ms < baseline.mid_ms) {
367+
} else if (result.avg_ms < baseline.avg_ms) {
364368
// Statistically significant improvement above threshold
365369
std::cout << "[Performance Improve " << std::abs(diff_percent) << "%] " << test_name << ": "
366-
<< std::setprecision(6) << "median: " << result.mid_ms << " ms "
367-
<< "(baseline: " << baseline.mid_ms << " ms, "
370+
<< std::setprecision(6) << "avg: " << result.avg_ms << " ms "
371+
<< "(baseline: " << baseline.avg_ms << " ms, "
368372
<< "t=" << t_stat << ", p=" << p_value << ")" << std::endl;
369373
} else {
370374
// Statistically significant regression above threshold - FAIL

0 commit comments

Comments
 (0)