Skip to content

Commit b84bd77

Browse files
Little-WallaceLittle-Wallace
committed
test: fix some perf context position
Signed-off-by: Little-Wallace <[email protected]>
1 parent 9da6f60 commit b84bd77

18 files changed

+95
-85
lines changed

‎db/c.cc

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@
1212
#include "rocksdb/c.h"
1313

1414
#include <stdlib.h>
15+
16+
#include <map>
17+
#include <unordered_set>
18+
#include <vector>
19+
1520
#include "port/port.h"
1621
#include "rocksdb/cache.h"
1722
#include "rocksdb/compaction_filter.h"
@@ -44,81 +49,77 @@
4449
#include "utilities/merge_operators.h"
4550
#include "utilities/rate_limiters/write_amp_based_rate_limiter.h"
4651

47-
#include <vector>
48-
#include <unordered_set>
49-
#include <map>
50-
52+
using rocksdb::BackupableDBOptions;
53+
using rocksdb::BackupEngine;
54+
using rocksdb::BackupID;
55+
using rocksdb::BackupInfo;
56+
using rocksdb::BatchResult;
57+
using rocksdb::BlockBasedTableOptions;
58+
using rocksdb::BottommostLevelCompaction;
5159
using rocksdb::BytewiseComparator;
5260
using rocksdb::Cache;
61+
using rocksdb::CheckPerfFlag;
62+
using rocksdb::Checkpoint;
5363
using rocksdb::ColumnFamilyDescriptor;
5464
using rocksdb::ColumnFamilyHandle;
5565
using rocksdb::ColumnFamilyOptions;
5666
using rocksdb::CompactionFilter;
5767
using rocksdb::CompactionFilterFactory;
5868
using rocksdb::CompactionOptionsFIFO;
69+
using rocksdb::CompactRangeOptions;
5970
using rocksdb::Comparator;
6071
using rocksdb::CompressionType;
61-
using rocksdb::WALRecoveryMode;
72+
using rocksdb::CuckooTableOptions;
6273
using rocksdb::DB;
6374
using rocksdb::DBOptions;
6475
using rocksdb::DbPath;
76+
using rocksdb::DisablePerfFlag;
77+
using rocksdb::EnablePerfFlag;
6578
using rocksdb::Env;
6679
using rocksdb::EnvOptions;
67-
using rocksdb::InfoLogLevel;
6880
using rocksdb::FileLock;
6981
using rocksdb::FilterPolicy;
7082
using rocksdb::FlushOptions;
83+
using rocksdb::InfoLogLevel;
7184
using rocksdb::IngestExternalFileOptions;
7285
using rocksdb::Iterator;
86+
using rocksdb::LiveFileMetaData;
7387
using rocksdb::Logger;
88+
using rocksdb::MemoryUtil;
7489
using rocksdb::MergeOperator;
7590
using rocksdb::MergeOperators;
7691
using rocksdb::NewBloomFilterPolicy;
92+
using rocksdb::NewGenericRateLimiter;
7793
using rocksdb::NewLRUCache;
94+
using rocksdb::NewWriteAmpBasedRateLimiter;
95+
using rocksdb::OptimisticTransactionDB;
96+
using rocksdb::OptimisticTransactionOptions;
7897
using rocksdb::Options;
79-
using rocksdb::BlockBasedTableOptions;
80-
using rocksdb::CuckooTableOptions;
98+
using rocksdb::PerfContext;
99+
using rocksdb::PerfLevel;
100+
using rocksdb::PinnableSlice;
81101
using rocksdb::RandomAccessFile;
82102
using rocksdb::Range;
103+
using rocksdb::RateLimiter;
83104
using rocksdb::ReadOptions;
105+
using rocksdb::RestoreOptions;
84106
using rocksdb::SequentialFile;
85107
using rocksdb::Slice;
86108
using rocksdb::SliceParts;
87109
using rocksdb::SliceTransform;
88110
using rocksdb::Snapshot;
89111
using rocksdb::SstFileWriter;
90112
using rocksdb::Status;
113+
using rocksdb::Transaction;
114+
using rocksdb::TransactionDB;
115+
using rocksdb::TransactionDBOptions;
116+
using rocksdb::TransactionLogIterator;
117+
using rocksdb::TransactionOptions;
118+
using rocksdb::WALRecoveryMode;
91119
using rocksdb::WritableFile;
92120
using rocksdb::WriteBatch;
93121
using rocksdb::WriteBatchWithIndex;
94122
using rocksdb::WriteOptions;
95-
using rocksdb::LiveFileMetaData;
96-
using rocksdb::BackupEngine;
97-
using rocksdb::BackupableDBOptions;
98-
using rocksdb::BackupInfo;
99-
using rocksdb::BackupID;
100-
using rocksdb::RestoreOptions;
101-
using rocksdb::CompactRangeOptions;
102-
using rocksdb::BottommostLevelCompaction;
103-
using rocksdb::RateLimiter;
104-
using rocksdb::NewGenericRateLimiter;
105-
using rocksdb::NewWriteAmpBasedRateLimiter;
106-
using rocksdb::PinnableSlice;
107-
using rocksdb::TransactionDBOptions;
108-
using rocksdb::TransactionDB;
109-
using rocksdb::TransactionOptions;
110-
using rocksdb::OptimisticTransactionDB;
111-
using rocksdb::OptimisticTransactionOptions;
112-
using rocksdb::Transaction;
113-
using rocksdb::Checkpoint;
114-
using rocksdb::TransactionLogIterator;
115-
using rocksdb::BatchResult;
116-
using rocksdb::PerfLevel;
117-
using rocksdb::EnablePerfFlag;
118-
using rocksdb::DisablePerfFlag;
119-
using rocksdb::CheckPerfFlag;
120-
using rocksdb::PerfContext;
121-
using rocksdb::MemoryUtil;
122123

123124
using std::shared_ptr;
124125
using std::vector;

‎db/column_family.cc

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -738,7 +738,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
738738
bool needed_delay = write_controller->NeedsDelay();
739739

740740
if (write_stall_condition == WriteStallCondition::kStopped &&
741-
write_stall_cause == WriteStallCause::kMemtableLimit && !mutable_cf_options.disable_write_stall) {
741+
write_stall_cause == WriteStallCause::kMemtableLimit &&
742+
!mutable_cf_options.disable_write_stall) {
742743
write_controller_token_ = write_controller->GetStopToken();
743744
internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
744745
ROCKS_LOG_WARN(
@@ -748,7 +749,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
748749
name_.c_str(), imm()->NumNotFlushed(),
749750
mutable_cf_options.max_write_buffer_number);
750751
} else if (write_stall_condition == WriteStallCondition::kStopped &&
751-
write_stall_cause == WriteStallCause::kL0FileCountLimit && !mutable_cf_options.disable_write_stall) {
752+
write_stall_cause == WriteStallCause::kL0FileCountLimit &&
753+
!mutable_cf_options.disable_write_stall) {
752754
write_controller_token_ = write_controller->GetStopToken();
753755
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1);
754756
if (compaction_picker_->IsLevel0CompactionInProgress()) {
@@ -759,7 +761,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
759761
"[%s] Stopping writes because we have %d level-0 files",
760762
name_.c_str(), vstorage->l0_delay_trigger_count());
761763
} else if (write_stall_condition == WriteStallCondition::kStopped &&
762-
write_stall_cause == WriteStallCause::kPendingCompactionBytes && !mutable_cf_options.disable_write_stall) {
764+
write_stall_cause == WriteStallCause::kPendingCompactionBytes &&
765+
!mutable_cf_options.disable_write_stall) {
763766
write_controller_token_ = write_controller->GetStopToken();
764767
internal_stats_->AddCFStats(
765768
InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
@@ -769,7 +772,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
769772
"bytes %" PRIu64,
770773
name_.c_str(), compaction_needed_bytes);
771774
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
772-
write_stall_cause == WriteStallCause::kMemtableLimit && !mutable_cf_options.disable_write_stall) {
775+
write_stall_cause == WriteStallCause::kMemtableLimit &&
776+
!mutable_cf_options.disable_write_stall) {
773777
write_controller_token_ =
774778
SetupDelay(write_controller, compaction_needed_bytes,
775779
prev_compaction_needed_bytes_, was_stopped,
@@ -784,7 +788,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
784788
mutable_cf_options.max_write_buffer_number,
785789
write_controller->delayed_write_rate());
786790
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
787-
write_stall_cause == WriteStallCause::kL0FileCountLimit && !mutable_cf_options.disable_write_stall) {
791+
write_stall_cause == WriteStallCause::kL0FileCountLimit &&
792+
!mutable_cf_options.disable_write_stall) {
788793
// L0 is the last two files from stopping.
789794
bool near_stop = vstorage->l0_delay_trigger_count() >=
790795
mutable_cf_options.level0_stop_writes_trigger - 2;
@@ -804,7 +809,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
804809
name_.c_str(), vstorage->l0_delay_trigger_count(),
805810
write_controller->delayed_write_rate());
806811
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
807-
write_stall_cause == WriteStallCause::kPendingCompactionBytes && !mutable_cf_options.disable_write_stall) {
812+
write_stall_cause == WriteStallCause::kPendingCompactionBytes &&
813+
!mutable_cf_options.disable_write_stall) {
808814
// If the distance to hard limit is less than 1/4 of the gap between soft
809815
// and
810816
// hard bytes limit, we think it is near stop and speed up the slowdown.
@@ -829,7 +835,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
829835
name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
830836
write_controller->delayed_write_rate());
831837
} else {
832-
assert(write_stall_condition == WriteStallCondition::kNormal || mutable_cf_options.disable_write_stall);
838+
assert(write_stall_condition == WriteStallCondition::kNormal ||
839+
mutable_cf_options.disable_write_stall);
833840
if (vstorage->l0_delay_trigger_count() >=
834841
GetL0ThresholdSpeedupCompaction(
835842
mutable_cf_options.level0_file_num_compaction_trigger,

‎db/db_impl/db_impl.cc

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
163163
seq_per_batch_(seq_per_batch),
164164
batch_per_txn_(batch_per_txn),
165165
db_lock_(nullptr),
166+
log_write_mutex_(stats_, env_, DB_MUTEX_WAIT_MICROS, false),
166167
shutting_down_(false),
167168
bg_cv_(&mutex_),
168169
logfile_number_(0),
@@ -1019,11 +1020,12 @@ Status DBImpl::SetDBOptions(
10191020
mutable_db_options_.max_background_jobs,
10201021
mutable_db_options_.base_background_compactions,
10211022
/* parallelize_compactions */ true);
1022-
const BGJobLimits new_bg_job_limits = GetBGJobLimits(
1023-
new_options.max_background_flushes,
1024-
new_options.max_background_compactions,
1025-
new_options.max_background_jobs,
1026-
new_options.base_background_compactions, /* parallelize_compactions */ true);
1023+
const BGJobLimits new_bg_job_limits =
1024+
GetBGJobLimits(new_options.max_background_flushes,
1025+
new_options.max_background_compactions,
1026+
new_options.max_background_jobs,
1027+
new_options.base_background_compactions,
1028+
/* parallelize_compactions */ true);
10271029

10281030
const bool max_flushes_increased =
10291031
new_bg_job_limits.max_flushes > current_bg_job_limits.max_flushes;

‎db/db_impl/db_impl_compaction_flush.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1988,7 +1988,8 @@ DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes,
19881988
}
19891989
if (!parallelize_compactions) {
19901990
// throttle background compactions until we deem necessary
1991-
res.max_compactions = std::max(1, std::min(base_background_compactions, res.max_compactions));
1991+
res.max_compactions =
1992+
std::max(1, std::min(base_background_compactions, res.max_compactions));
19921993
}
19931994
return res;
19941995
}

‎db/db_impl/db_impl_open.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
5757
}
5858
auto bg_job_limits = DBImpl::GetBGJobLimits(
5959
result.max_background_flushes, result.max_background_compactions,
60-
result.max_background_jobs, result.base_background_compactions, true /* parallelize_compactions */);
60+
result.max_background_jobs, result.base_background_compactions,
61+
true /* parallelize_compactions */);
6162
result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions,
6263
Env::Priority::LOW);
6364
result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes,

‎db/db_impl/db_impl_write.cc

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -88,15 +88,15 @@ Status DBImpl::MultiBatchWriteImpl(const WriteOptions& write_options,
8888
WriteContext write_context;
8989
bool ignore_missing_faimly = write_options.ignore_missing_column_families;
9090
if (writer.state == WriteThread::STATE_GROUP_LEADER) {
91+
PERF_TIMER_STOP(write_pre_and_post_process_time);
92+
PERF_TIMER_GUARD(write_delay_time);
9193
if (writer.callback && !writer.callback->AllowWriteBatching()) {
9294
write_thread_.WaitForMemTableWriters();
9395
}
9496
WriteThread::WriteGroup wal_write_group;
9597
LogContext log_context;
96-
PERF_TIMER_STOP(write_pre_and_post_process_time);
9798
writer.status =
9899
PreprocessWrite(write_options, &log_context, &write_context);
99-
PERF_TIMER_START(write_pre_and_post_process_time);
100100

101101
// This can set non-OK status if callback fail.
102102
last_batch_group_size_ =
@@ -132,7 +132,6 @@ Status DBImpl::MultiBatchWriteImpl(const WriteOptions& write_options,
132132
RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
133133
RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size);
134134

135-
PERF_TIMER_STOP(write_pre_and_post_process_time);
136135
if (!write_options.disableWAL) {
137136
PERF_TIMER_GUARD(write_wal_time);
138137
stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1);
@@ -163,14 +162,14 @@ Status DBImpl::MultiBatchWriteImpl(const WriteOptions& write_options,
163162
bool is_leader_thread = false;
164163
WriteThread::WriteGroup memtable_write_group;
165164
if (writer.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) {
166-
PERF_TIMER_GUARD(write_memtable_time);
167165
assert(writer.ShouldWriteToMemtable());
168166
write_thread_.EnterAsMemTableWriter(&writer, &memtable_write_group);
169167
assert(immutable_db_options_.allow_concurrent_memtable_write);
170168
if (memtable_write_group.size > 1) {
171169
is_leader_thread = true;
172170
write_thread_.LaunchParallelMemTableWriters(&memtable_write_group);
173171
} else {
172+
PERF_TIMER_GUARD(write_memtable_time);
174173
auto version_set = versions_->GetColumnFamilySet();
175174
memtable_write_group.running.store(0);
176175
for (auto it = memtable_write_group.begin();
@@ -194,6 +193,7 @@ Status DBImpl::MultiBatchWriteImpl(const WriteOptions& write_options,
194193
}
195194
if (writer.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
196195
assert(writer.ShouldWriteToMemtable());
196+
PERF_TIMER_GUARD(write_memtable_time);
197197
auto version_set = versions_->GetColumnFamilySet();
198198
WriteBatchInternal::AsyncInsertInto(
199199
&writer, writer.sequence, version_set, &flush_scheduler_,
@@ -640,8 +640,8 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
640640
LogContext log_context(!write_options.disableWAL && write_options.sync);
641641
// PreprocessWrite does its own perf timing.
642642
PERF_TIMER_STOP(write_pre_and_post_process_time);
643+
PERF_TIMER_GUARD(write_delay_time);
643644
w.status = PreprocessWrite(write_options, &log_context, &write_context);
644-
PERF_TIMER_START(write_pre_and_post_process_time);
645645

646646
// This can set non-OK status if callback fail.
647647
last_batch_group_size_ =
@@ -678,8 +678,6 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
678678
RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
679679
RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size);
680680

681-
PERF_TIMER_STOP(write_pre_and_post_process_time);
682-
683681
if (w.status.ok() && !write_options.disableWAL) {
684682
PERF_TIMER_GUARD(write_wal_time);
685683
stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1);
@@ -752,7 +750,7 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
752750
WriteCallback* callback, uint64_t log_ref,
753751
SequenceNumber seq,
754752
const size_t sub_batch_cnt) {
755-
PERF_TIMER_GUARD(write_pre_and_post_process_time);
753+
PERF_TIMER_GUARD(write_memtable_time);
756754
StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
757755

758756
WriteThread::Writer w(write_options, my_batch, callback, log_ref,
@@ -824,6 +822,8 @@ Status DBImpl::WriteImplWALOnly(
824822
// else we are the leader of the write batch group
825823
assert(w.state == WriteThread::STATE_GROUP_LEADER);
826824

825+
PERF_TIMER_STOP(write_pre_and_post_process_time);
826+
PERF_TIMER_GUARD(write_delay_time);
827827
if (publish_last_seq == kDoPublishLastSeq) {
828828
// Currently we only use kDoPublishLastSeq in unordered_write
829829
assert(immutable_db_options_.unordered_write);
@@ -884,8 +884,6 @@ Status DBImpl::WriteImplWALOnly(
884884
}
885885
RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size);
886886

887-
PERF_TIMER_STOP(write_pre_and_post_process_time);
888-
889887
PERF_TIMER_GUARD(write_wal_time);
890888
// LastAllocatedSequence is increased inside WriteToWAL under
891889
// wal_write_mutex_ to ensure ordered events in WAL
@@ -934,7 +932,6 @@ Status DBImpl::WriteImplWALOnly(
934932
status = SyncWAL();
935933
}
936934
}
937-
PERF_TIMER_START(write_pre_and_post_process_time);
938935

939936
if (!w.CallbackFailed()) {
940937
WriteStatusCheck(status);
@@ -1036,19 +1033,15 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
10361033
}
10371034

10381035
PERF_TIMER_STOP(write_scheduling_flushes_compactions_time);
1039-
PERF_TIMER_GUARD(write_pre_and_post_process_time);
10401036

10411037
if (UNLIKELY(status.ok() && (write_controller_.IsStopped() ||
10421038
write_controller_.NeedsDelay()))) {
1043-
PERF_TIMER_STOP(write_pre_and_post_process_time);
1044-
PERF_TIMER_GUARD(write_delay_time);
10451039
// We don't know size of curent batch so that we always use the size
10461040
// for previous one. It might create a fairness issue that expiration
10471041
// might happen for smaller writes but larger writes can go through.
10481042
// Can optimize it if it is an issue.
10491043
InstrumentedMutexLock l(&mutex_);
10501044
status = DelayWrite(last_batch_group_size_, write_options);
1051-
PERF_TIMER_START(write_pre_and_post_process_time);
10521045
}
10531046

10541047
InstrumentedMutexLock l(&log_write_mutex_);
@@ -1634,7 +1627,6 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
16341627
// is that in case the write is heavy, low pri writes may never have
16351628
// a chance to run. Now we guarantee we are still slowly making
16361629
// progress.
1637-
PERF_TIMER_GUARD(write_delay_time);
16381630
write_controller_.low_pri_rate_limiter()->Request(
16391631
my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
16401632
RateLimiter::OpType::kWrite);

0 commit comments

Comments
 (0)