Skip to content

Commit 0f12bbe

Browse files
lucagiac81lucagiac81
committed
Separate derived compression settings
1 parent 45a3eca commit 0f12bbe

17 files changed

+105
-90
lines changed

‎db/blob/blob_file_builder.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ BlobFileBuilder::BlobFileBuilder(
6666
immutable_options_(immutable_options),
6767
min_blob_size_(mutable_cf_options->min_blob_size),
6868
blob_file_size_(mutable_cf_options->blob_file_size),
69-
blob_compressor_(mutable_cf_options->blob_compressor),
69+
blob_compressor_(mutable_cf_options->derived_blob_compressor),
7070
prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache),
7171
file_options_(file_options),
7272
db_id_(std::move(db_id)),

‎db/column_family.cc

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -130,17 +130,18 @@ void GetIntTblPropCollectorFactory(
130130
Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
131131
MutableCFOptions moptions(cf_options);
132132
ImmutableCFOptions ioptions(cf_options);
133-
if (moptions.compressor && !moptions.compressor->Supported()) {
133+
if (moptions.derived_compressor &&
134+
!moptions.derived_compressor->Supported()) {
134135
return Status::InvalidArgument("Compression type " +
135-
moptions.compressor->GetId() +
136+
moptions.derived_compressor->GetId() +
136137
" is not linked with the binary.");
137-
} else if (moptions.bottommost_compressor &&
138-
!moptions.bottommost_compressor->Supported()) {
139-
return Status::InvalidArgument("Compression type " +
140-
moptions.bottommost_compressor->GetId() +
141-
" is not linked with the binary.");
142-
} else if (!moptions.compressor_per_level.empty()) {
143-
for (const auto& compressor : moptions.compressor_per_level) {
138+
} else if (moptions.derived_bottommost_compressor &&
139+
!moptions.derived_bottommost_compressor->Supported()) {
140+
return Status::InvalidArgument(
141+
"Compression type " + moptions.derived_bottommost_compressor->GetId() +
142+
" is not linked with the binary.");
143+
} else if (!moptions.derived_compressor_per_level.empty()) {
144+
for (const auto& compressor : moptions.derived_compressor_per_level) {
144145
if (compressor == nullptr) {
145146
return Status::InvalidArgument("Compression type is invalid.");
146147
} else if (!compressor->Supported()) {
@@ -170,9 +171,10 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
170171
"should be nonzero if we're using zstd's dictionary generator.");
171172
}
172173
}
173-
if (moptions.blob_compressor && !moptions.blob_compressor->Supported()) {
174+
if (moptions.derived_blob_compressor &&
175+
!moptions.derived_blob_compressor->Supported()) {
174176
return Status::InvalidArgument("Blob compression type " +
175-
moptions.blob_compressor->GetId() +
177+
moptions.derived_blob_compressor->GetId() +
176178
" is not linked with the binary.");
177179
}
178180
return Status::OK();

‎db/compaction/compaction_picker.cc

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,12 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
8787
// If bottommost_compression is set and we are compacting to the
8888
// bottommost level then we should use it.
8989
bool bottom_level = (level >= (vstorage->num_non_empty_levels() - 1));
90-
if (moptions.bottommost_compressor != nullptr && bottom_level) {
91-
return moptions.bottommost_compressor;
90+
if (moptions.derived_bottommost_compressor != nullptr && bottom_level) {
91+
return moptions.derived_bottommost_compressor;
9292
}
9393
// If the user has specified a different compression level for each level,
9494
// then pick the compression for that level.
95-
if (!moptions.compressor_per_level.empty()) {
95+
if (!moptions.derived_compressor_per_level.empty()) {
9696
// It is possible for level_ to be -1; in that case, we use level
9797
// 0's compression. This occurs mostly in backwards compatibility
9898
// situations when the builder doesn't know what level the file
@@ -101,7 +101,8 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
101101
assert(level == 0 || level >= base_level);
102102
int lvl = std::max(0, level - base_level + 1);
103103
int idx = std::min(
104-
static_cast<int>(moptions.compressor_per_level.size()) - 1, lvl);
104+
static_cast<int>(moptions.derived_compressor_per_level.size()) - 1,
105+
lvl);
105106
// If not specified directly by the user, compressors in
106107
// compressor_per_level are instantiated using compression_opts. If the user
107108
// enabled bottommost_compression_opts, we need to create a new compressor
@@ -112,10 +113,10 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
112113
moptions.compression_per_level[idx],
113114
moptions.bottommost_compression_opts);
114115
} else {
115-
return moptions.compressor_per_level[idx];
116+
return moptions.derived_compressor_per_level[idx];
116117
}
117118
} else {
118-
return moptions.compressor;
119+
return moptions.derived_compressor;
119120
}
120121
}
121122

‎db/compaction/compaction_picker_fifo.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
181181
vstorage, ioptions_, mutable_cf_options, mutable_db_options,
182182
{comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,
183183
0 /* max compaction bytes, not applicable */,
184-
0 /* output path ID */, mutable_cf_options.compressor,
184+
0 /* output path ID */, mutable_cf_options.derived_compressor,
185185
Temperature::kUnknown, 0 /* max_subcompactions */, {},
186186
/* is manual */ false, /* trim_ts */ "",
187187
vstorage->CompactionScore(0),
@@ -416,7 +416,7 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
416416
vstorage, ioptions_, mutable_cf_options, mutable_db_options,
417417
std::move(inputs), 0, 0 /* output file size limit */,
418418
0 /* max compaction bytes, not applicable */, 0 /* output path ID */,
419-
mutable_cf_options.compressor, compaction_target_temp,
419+
mutable_cf_options.derived_compressor, compaction_target_temp,
420420
/* max_subcompactions */ 0, {}, /* is manual */ false, /* trim_ts */ "",
421421
vstorage->CompactionScore(0),
422422
/* is deletion compaction */ false, /* l0_files_might_overlap */ true,

‎db/db_impl/db_impl.cc

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,14 +138,14 @@ std::shared_ptr<Compressor> GetCompressionFlush(
138138
// latency overhead is not offset by saving much space.
139139
if (ioptions.compaction_style == kCompactionStyleUniversal) {
140140
if (moptions.compaction_options_universal.compression_size_percent < 0) {
141-
return moptions.compressor;
141+
return moptions.derived_compressor;
142142
} else {
143143
return BuiltinCompressor::GetCompressor(kNoCompression);
144144
}
145-
} else if (moptions.compressor_per_level.empty()) {
146-
return moptions.compressor;
145+
} else if (moptions.derived_compressor_per_level.empty()) {
146+
return moptions.derived_compressor;
147147
} else {
148-
return moptions.compressor_per_level[0];
148+
return moptions.derived_compressor_per_level[0];
149149
}
150150
}
151151

‎db/db_impl/db_impl_compaction_flush.cc

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1931,8 +1931,9 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
19311931
->compaction_style) /* output file size limit, not applicable */
19321932
,
19331933
LLONG_MAX /* max compaction bytes, not applicable */,
1934-
0 /* output path ID, not applicable */, mutable_cf_options.compressor,
1935-
Temperature::kUnknown, 0 /* max_subcompactions, not applicable */,
1934+
0 /* output path ID, not applicable */,
1935+
mutable_cf_options.derived_compressor, Temperature::kUnknown,
1936+
0 /* max_subcompactions, not applicable */,
19361937
{} /* grandparents, not applicable */, false /* is manual */,
19371938
"" /* trim_ts */, -1 /* score, not applicable */,
19381939
false /* is deletion compaction, not applicable */,
@@ -4155,7 +4156,7 @@ void DBImpl::BuildCompactionJobInfo(
41554156
newf.first, file_number, meta.oldest_blob_file_number});
41564157
}
41574158
compaction_job_info->blob_compression_type =
4158-
c->mutable_cf_options()->blob_compressor->GetCompressionType();
4159+
c->mutable_cf_options()->derived_blob_compressor->GetCompressionType();
41594160

41604161
// Update BlobFilesInfo.
41614162
for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) {

‎db/external_sst_file_ingestion_job.cc

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -550,8 +550,9 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
550550
*/
551551
,
552552
LLONG_MAX /* max compaction bytes, not applicable */,
553-
0 /* output path ID, not applicable */, mutable_cf_options.compressor,
554-
Temperature::kUnknown, 0 /* max_subcompaction, not applicable */,
553+
0 /* output path ID, not applicable */,
554+
mutable_cf_options.derived_compressor, Temperature::kUnknown,
555+
0 /* max_subcompaction, not applicable */,
555556
{} /* grandparents, not applicable */, false /* is manual */,
556557
"" /* trim_ts */, -1 /* score, not applicable */,
557558
false /* is deletion compaction, not applicable */,

‎db/flush_job.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1134,7 +1134,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {
11341134
info->table_properties = table_properties_;
11351135
info->flush_reason = flush_reason_;
11361136
info->blob_compression_type =
1137-
mutable_cf_options_.blob_compressor->GetCompressionType();
1137+
mutable_cf_options_.derived_blob_compressor->GetCompressionType();
11381138

11391139
// Update BlobFilesInfo.
11401140
for (const auto& blob_file : edit_->GetBlobFileAdditions()) {

‎db/table_properties_collector_test.cc

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,10 @@ void MakeBuilder(
5151
std::unique_ptr<FSWritableFile> wf(new test::StringSink);
5252
writable->reset(
5353
new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions()));
54-
TableBuilderOptions tboptions(ioptions, moptions, internal_comparator,
55-
int_tbl_prop_collector_factories,
56-
moptions.compressor, kTestColumnFamilyId,
57-
kTestColumnFamilyName, kTestLevel);
54+
TableBuilderOptions tboptions(
55+
ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories,
56+
moptions.derived_compressor, kTestColumnFamilyId, kTestColumnFamilyName,
57+
kTestLevel);
5858
builder->reset(NewTableBuilder(tboptions, writable->get()));
5959
}
6060
} // namespace

‎options/cf_options.cc

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,41 +1211,54 @@ void MutableCFOptions::RefreshDerivedOptions(int num_levels,
12111211
}
12121212
}
12131213

1214+
derived_compressor.reset();
12141215
if (compressor == nullptr) {
1215-
compressor =
1216+
derived_compressor =
12161217
BuiltinCompressor::GetCompressor(compression, compression_opts);
1217-
}
1218-
if (compressor == nullptr) {
1219-
compressor = BuiltinCompressor::GetCompressor(kSnappyCompression);
1218+
if (derived_compressor == nullptr) {
1219+
derived_compressor = BuiltinCompressor::GetCompressor(kSnappyCompression);
1220+
}
1221+
} else {
1222+
derived_compressor = compressor;
12201223
}
12211224

1225+
derived_bottommost_compressor.reset();
12221226
if (bottommost_compressor == nullptr) {
12231227
if (bottommost_compression != kDisableCompressionOption) {
12241228
if (bottommost_compression_opts.enabled) {
1225-
bottommost_compressor = BuiltinCompressor::GetCompressor(
1229+
derived_bottommost_compressor = BuiltinCompressor::GetCompressor(
12261230
bottommost_compression, bottommost_compression_opts);
12271231
} else {
1228-
bottommost_compressor = BuiltinCompressor::GetCompressor(
1232+
derived_bottommost_compressor = BuiltinCompressor::GetCompressor(
12291233
bottommost_compression, compression_opts);
12301234
}
12311235
}
1236+
} else {
1237+
derived_bottommost_compressor = bottommost_compressor;
12321238
}
12331239

1240+
derived_blob_compressor.reset();
12341241
if (blob_compressor == nullptr) {
12351242
if (blob_compression_type != kDisableCompressionOption) {
1236-
blob_compressor = BuiltinCompressor::GetCompressor(blob_compression_type,
1237-
compression_opts);
1243+
derived_blob_compressor = BuiltinCompressor::GetCompressor(
1244+
blob_compression_type, compression_opts);
12381245
}
1239-
}
1240-
if (blob_compressor == nullptr) {
1241-
blob_compressor = BuiltinCompressor::GetCompressor(kNoCompression);
1246+
if (derived_blob_compressor == nullptr) {
1247+
derived_blob_compressor =
1248+
BuiltinCompressor::GetCompressor(kNoCompression);
1249+
}
1250+
} else {
1251+
derived_blob_compressor = blob_compressor;
12421252
}
12431253

1254+
derived_compressor_per_level.clear();
12441255
if (compressor_per_level.empty()) {
12451256
for (auto type : compression_per_level) {
1246-
compressor_per_level.push_back(
1257+
derived_compressor_per_level.push_back(
12471258
BuiltinCompressor::GetCompressor(type, compression_opts));
12481259
}
1260+
} else {
1261+
derived_compressor_per_level = compressor_per_level;
12491262
}
12501263
}
12511264

‎options/cf_options.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,10 @@ struct MutableCFOptions {
230230
// Derived options
231231
// Per-level target file size.
232232
std::vector<uint64_t> max_file_size;
233+
std::shared_ptr<Compressor> derived_compressor;
234+
std::shared_ptr<Compressor> derived_bottommost_compressor;
235+
std::shared_ptr<Compressor> derived_blob_compressor;
236+
std::vector<std::shared_ptr<Compressor>> derived_compressor_per_level;
233237
};
234238

235239
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);

‎options/options.cc

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -188,14 +188,17 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
188188
MutableCFOptions moptions(*this);
189189
ConfigOptions config_options;
190190

191-
if (!moptions.compressor_per_level.empty()) {
192-
for (unsigned int i = 0; i < moptions.compressor_per_level.size(); i++) {
193-
ROCKS_LOG_HEADER(log, " Options.compression[%d]: %s", i,
194-
moptions.compressor_per_level[i]->GetId().c_str());
191+
if (!moptions.derived_compressor_per_level.empty()) {
192+
for (unsigned int i = 0; i < moptions.derived_compressor_per_level.size();
193+
i++) {
194+
ROCKS_LOG_HEADER(
195+
log, " Options.compression[%d]: %s", i,
196+
moptions.derived_compressor_per_level[i]->GetId().c_str());
195197
}
196-
} else if (moptions.compressor) {
197-
ROCKS_LOG_HEADER(log, " Options.compression: %s",
198-
moptions.compressor->ToString(config_options).c_str());
198+
} else if (moptions.derived_compressor) {
199+
ROCKS_LOG_HEADER(
200+
log, " Options.compression: %s",
201+
moptions.derived_compressor->ToString(config_options).c_str());
199202
} else {
200203
ROCKS_LOG_HEADER(
201204
log, " Options.compression: %s",
@@ -228,10 +231,11 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
228231
"%" PRIu64,
229232
compression_opts.max_dict_buffer_bytes);
230233
}
231-
if (moptions.bottommost_compressor) {
234+
if (moptions.derived_bottommost_compressor) {
232235
ROCKS_LOG_HEADER(
233236
log, " Options.bottommost_compression: %s",
234-
moptions.bottommost_compressor->ToString(config_options).c_str());
237+
moptions.derived_bottommost_compressor->ToString(config_options)
238+
.c_str());
235239
} else {
236240
ROCKS_LOG_HEADER(
237241
log, " Options.bottommost_compression: %s",
@@ -451,9 +455,9 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
451455
ROCKS_LOG_HEADER(log,
452456
" Options.blob_file_size: %" PRIu64,
453457
blob_file_size);
454-
if (moptions.blob_compressor) {
458+
if (moptions.derived_blob_compressor) {
455459
ROCKS_LOG_HEADER(log, " Options.blob_compression: %s",
456-
moptions.blob_compressor->GetId().c_str());
460+
moptions.derived_blob_compressor->GetId().c_str());
457461
} else {
458462
ROCKS_LOG_HEADER(
459463
log, " Options.blob_compression_type: %s",

‎options/options_settable_test.cc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
646646
sizeof(std::vector<std::shared_ptr<Compressor>>)},
647647
{offsetof(struct MutableCFOptions, max_file_size),
648648
sizeof(std::vector<uint64_t>)},
649+
{offsetof(struct MutableCFOptions, derived_compressor_per_level),
650+
sizeof(std::vector<std::shared_ptr<Compressor>>)},
649651
};
650652

651653
// For all memory used for options, pre-fill every char. Otherwise, the

‎table/block_based/data_block_hash_index_test.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -556,7 +556,7 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
556556
builder.reset(ioptions.table_factory->NewTableBuilder(
557557
TableBuilderOptions(
558558
ioptions, moptions, internal_comparator,
559-
&int_tbl_prop_collector_factories, moptions.compressor,
559+
&int_tbl_prop_collector_factories, moptions.derived_compressor,
560560
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
561561
column_family_name, level_),
562562
file_writer.get()));

‎table/sst_file_writer.cc

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -311,14 +311,14 @@ Status SstFileWriter::Open(const std::string& file_path) {
311311
sst_file->SetIOPriority(r->io_priority);
312312

313313
std::shared_ptr<Compressor> compressor;
314-
if (r->mutable_cf_options.bottommost_compressor != nullptr) {
315-
compressor = r->mutable_cf_options.bottommost_compressor;
316-
} else if (r->mutable_cf_options.compressor_per_level.empty()) {
317-
compressor = r->mutable_cf_options.compressor;
314+
if (r->mutable_cf_options.derived_bottommost_compressor != nullptr) {
315+
compressor = r->mutable_cf_options.derived_bottommost_compressor;
316+
} else if (r->mutable_cf_options.derived_compressor_per_level.empty()) {
317+
compressor = r->mutable_cf_options.derived_compressor;
318318
} else {
319319
// Use the compression of the last level if we have per level compression
320-
auto levels = r->mutable_cf_options.compressor_per_level.size();
321-
compressor = r->mutable_cf_options.compressor_per_level[levels - 1];
320+
auto levels = r->mutable_cf_options.derived_compressor_per_level.size();
321+
compressor = r->mutable_cf_options.derived_compressor_per_level[levels - 1];
322322
}
323323

324324
IntTblPropCollectorFactories int_tbl_prop_collector_factories;

‎table/table_test.cc

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ class TableConstructor : public Constructor {
400400
builder.reset(ioptions.table_factory->NewTableBuilder(
401401
TableBuilderOptions(ioptions, moptions, internal_comparator,
402402
&int_tbl_prop_collector_factories,
403-
moptions.compressor, kUnknownColumnFamily,
403+
moptions.derived_compressor, kUnknownColumnFamily,
404404
column_family_name, level_),
405405
file_writer_.get()));
406406

@@ -3995,9 +3995,10 @@ TEST_P(BlockBasedTableTest, NoFileChecksum) {
39953995
f.CreateWritableFile();
39963996
std::unique_ptr<TableBuilder> builder;
39973997
builder.reset(ioptions.table_factory->NewTableBuilder(
3998-
TableBuilderOptions(
3999-
ioptions, moptions, *comparator, &int_tbl_prop_collector_factories,
4000-
moptions.compressor, kUnknownColumnFamily, column_family_name, level),
3998+
TableBuilderOptions(ioptions, moptions, *comparator,
3999+
&int_tbl_prop_collector_factories,
4000+
moptions.derived_compressor, kUnknownColumnFamily,
4001+
column_family_name, level),
40014002
f.GetFileWriter()));
40024003
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
40034004
f.AddKVtoKVMap(1000);
@@ -4030,9 +4031,10 @@ TEST_P(BlockBasedTableTest, Crc32cFileChecksum) {
40304031
f.SetFileChecksumGenerator(checksum_crc32c_gen1.release());
40314032
std::unique_ptr<TableBuilder> builder;
40324033
builder.reset(ioptions.table_factory->NewTableBuilder(
4033-
TableBuilderOptions(
4034-
ioptions, moptions, *comparator, &int_tbl_prop_collector_factories,
4035-
moptions.compressor, kUnknownColumnFamily, column_family_name, level),
4034+
TableBuilderOptions(ioptions, moptions, *comparator,
4035+
&int_tbl_prop_collector_factories,
4036+
moptions.derived_compressor, kUnknownColumnFamily,
4037+
column_family_name, level),
40364038
f.GetFileWriter()));
40374039
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
40384040
f.AddKVtoKVMap(1000);

0 commit comments

Comments
 (0)