Skip to content

Commit f8a4034

Browse files
authored
Merge pull request #323 from eseiler/final/dynamic
feat: final dynamic code
2 parents 1e3e168 + 53161d7 commit f8a4034

12 files changed

Lines changed: 230 additions & 29 deletions

‎include/hibf/interleaved_bloom_filter.hpp‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,24 @@ struct bin_index
8080
size_t value;
8181
};
8282

83+
/*!\brief A strong type that represents the empty bin fraction for the seqan::hibf::interleaved_bloom_filter.
84+
* \ingroup ibf
85+
* \qualifier strong
86+
*/
87+
struct empty_bin_fraction
88+
{
89+
double value;
90+
};
91+
92+
/*!\brief A strong type that represents whether to track occupancy for the seqan::hibf::interleaved_bloom_filter.
93+
* \ingroup ibf
94+
* \qualifier strong
95+
*/
96+
struct track_occupancy
97+
{
98+
bool value;
99+
};
100+
83101
/*!\brief The IBF binning directory. A data structure that efficiently answers set-membership queries for multiple bins.
84102
* \ingroup ibf
85103
* \implements seqan::hibf::cerealisable
@@ -214,18 +232,26 @@ class interleaved_bloom_filter : private seqan::hibf::bit_vector
214232
* \param bins_ The number of bins.
215233
* \param size The bitvector size.
216234
* \param funs The number of hash functions. Default 2. At least 1, at most 5.
217-
* \param track_occupancy_ Whether to track the occupancy of the bins.
235+
* \param empty_bin_fraction The fraction of total technical bins that should be empty.
236+
* \param track_occupancy Whether to track the occupancy of the bins.
218237
*
219238
* \details
220239
*
240+
* Upon construction, `bins_` many bins are immediately accessible.
241+
* Additionally, enough space is reserve to accomodate a total of at least `bins_ / empty_bin_fraction` many bins.
242+
* For example, with `bins_ = 64` and `empty_bin_fraction = 0.5`, the Interleaved Bloom Filter will reserve space
243+
* for 128 bins, 64 of which are immediately accessible and 64 of which are reserved for future use.
244+
*
221245
* ### Example
222246
*
223247
* \include test/snippet/ibf/interleaved_bloom_filter_constructor.cpp
224248
*/
225-
interleaved_bloom_filter(seqan::hibf::bin_count const bins_,
226-
seqan::hibf::bin_size const size,
227-
seqan::hibf::hash_function_count const funs = seqan::hibf::hash_function_count{2u},
228-
bool const track_occupancy_ = false);
249+
interleaved_bloom_filter(
250+
seqan::hibf::bin_count const bins_,
251+
seqan::hibf::bin_size const size,
252+
seqan::hibf::hash_function_count const funs = seqan::hibf::hash_function_count{2u},
253+
seqan::hibf::empty_bin_fraction const empty_bin_fraction = seqan::hibf::empty_bin_fraction{0.0},
254+
seqan::hibf::track_occupancy const track_occupancy = seqan::hibf::track_occupancy{false});
229255

230256
/*!\brief Construct an Interleaved Bloom Filter.
231257
* \param configuration The seqan::hibf::config.
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// SPDX-FileCopyrightText: 2006-2025, Knut Reinert & Freie Universität Berlin
2+
// SPDX-FileCopyrightText: 2016-2025, Knut Reinert & MPI für molekulare Genetik
3+
// SPDX-License-Identifier: BSD-3-Clause
4+
5+
#pragma once
6+
7+
#include <cmath> // for floor
8+
#include <cstddef> // for size_t
9+
10+
#include <hibf/platform.hpp>
11+
12+
namespace seqan::hibf
13+
{
14+
15+
/*!\brief Returns the total number of bins such that a `fraction` of bins is empty, and `tmax` many bins are non-empty.
16+
* \param[in] tmax The number of non-empty bins.
17+
* \param[in] fraction The fraction of the total number of bins that should be empty.
18+
* \ingroup hibf
19+
*/
20+
[[nodiscard]] constexpr size_t add_empty_bins(size_t const tmax, double const fraction) noexcept
21+
{
22+
return std::floor(tmax / (1.0 - fraction));
23+
}
24+
25+
} // namespace seqan::hibf

‎include/hibf/misc/subtract_empty_bins.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#pragma once
66

77
#include <algorithm> // for clamp
8+
#include <cmath> // for ceil
89
#include <cstddef> // for size_t
910

1011
#include <hibf/platform.hpp>
@@ -25,7 +26,7 @@ namespace seqan::hibf
2526
if (fraction == 0.0 || tmax <= 2u)
2627
return tmax;
2728

28-
size_t const number_of_empty_bins = std::clamp<size_t>(tmax * fraction, 1, tmax - 2);
29+
size_t const number_of_empty_bins = std::clamp<size_t>(std::ceil(tmax * fraction), 1, tmax - 2);
2930
return tmax - number_of_empty_bins;
3031
}
3132

‎src/build/construct_ibf.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ seqan::hibf::interleaved_bloom_filter construct_ibf(robin_hood::unordered_flat_s
5151
seqan::hibf::interleaved_bloom_filter ibf{bin_count,
5252
bin_size,
5353
seqan::hibf::hash_function_count{data.config.number_of_hash_functions},
54-
data.config.track_occupancy};
54+
seqan::hibf::empty_bin_fraction{data.config.empty_bin_fraction},
55+
seqan::hibf::track_occupancy{data.config.track_occupancy}};
5556

5657
local_index_allocation_timer.stop();
5758
data.index_allocation_timer += local_index_allocation_timer;

‎src/config.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <hibf/config.hpp> // for config
1818
#include <hibf/layout/prefixes.hpp> // for meta_header, meta_hibf_config_end, meta_hibf_config_start
1919
#include <hibf/misc/next_multiple_of_64.hpp> // for next_multiple_of_64
20+
#include <hibf/misc/subtract_empty_bins.hpp> // for subtract_empty_bins
2021

2122
namespace seqan::hibf
2223
{
@@ -117,7 +118,11 @@ void config::validate_and_set_defaults()
117118
throw std::invalid_argument{"[HIBF CONFIG ERROR] config::empty_bin_fraction must be in [0.0,1.0)."};
118119

119120
if (empty_bin_fraction != 0.0)
121+
{
120122
track_occupancy = true;
123+
size_t const subtracted = seqan::hibf::subtract_empty_bins(tmax, empty_bin_fraction);
124+
empty_bin_fraction = 1.0 - static_cast<double>(subtracted) / tmax;
125+
}
121126

122127
if (alpha < 0.0)
123128
throw std::invalid_argument{"[HIBF CONFIG ERROR] config::alpha must be positive."};

‎src/hierarchical_interleaved_bloom_filter.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ size_t hierarchical_build(hierarchical_interleaved_bloom_filter & hibf,
4242
layout::graph::node const & current_node,
4343
build::build_data & data,
4444
bool is_root,
45-
size_t const parent_ibf_idx = 0u)
45+
size_t const = 0)
4646
{
4747
size_t const ibf_pos{data.request_ibf_idx()};
4848

@@ -63,13 +63,15 @@ size_t hierarchical_build(hierarchical_interleaved_bloom_filter & hibf,
6363
if (current_node.max_bin_is_merged())
6464
{
6565
// recursively initialize favourite child first
66-
technical_bin_to_ibf_id[current_node.max_bin_index] =
66+
size_t const favourite_child_ibf_idx =
6767
hierarchical_build(hibf,
6868
kmers,
6969
current_node.children[current_node.favourite_child_idx.value()],
7070
data,
7171
false,
7272
ibf_pos);
73+
technical_bin_to_ibf_id[current_node.max_bin_index] = favourite_child_ibf_idx;
74+
hibf.prev_ibf_id[favourite_child_ibf_idx] = {.ibf_idx = ibf_pos, .bin_idx = current_node.max_bin_index};
7375
return 1;
7476
}
7577
else // max bin is not a merged bin
@@ -127,7 +129,7 @@ size_t hierarchical_build(hierarchical_interleaved_bloom_filter & hibf,
127129
robin_hood::unordered_flat_set<uint64_t> local_kmers{};
128130
size_t const local_ibf_pos = hierarchical_build(hibf, local_kmers, child, data, false, ibf_pos);
129131
auto parent_bin_index = child.parent_bin_index;
130-
hibf.prev_ibf_id[local_ibf_pos] = {.ibf_idx = parent_ibf_idx, .bin_idx = parent_bin_index};
132+
hibf.prev_ibf_id[local_ibf_pos] = {.ibf_idx = ibf_pos, .bin_idx = parent_bin_index};
131133
{
132134
size_t const mutex_id{parent_bin_index / 64};
133135
std::lock_guard<std::mutex> guard{local_ibf_mutex[mutex_id]};

‎src/interleaved_bloom_filter.cpp‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <hibf/config.hpp> // for config, insert_iterator
1616
#include <hibf/contrib/robin_hood.hpp> // for hash, unordered_flat_set
1717
#include <hibf/interleaved_bloom_filter.hpp> // for interleaved_bloom_filter, bin_count, bin_index, bin_size, hash_...
18+
#include <hibf/misc/add_empty_bins.hpp> // for add_empty_bins
1819
#include <hibf/misc/bit_vector.hpp> // for bit_vector
1920
#include <hibf/misc/divide_and_ceil.hpp> // for divide_and_ceil
2021
#include <hibf/misc/unreachable.hpp> // for assert, unreachable
@@ -32,11 +33,12 @@ namespace seqan::hibf
3233
interleaved_bloom_filter::interleaved_bloom_filter(seqan::hibf::bin_count const bins_,
3334
seqan::hibf::bin_size const size,
3435
seqan::hibf::hash_function_count const funs,
35-
bool const track_occupancy_) :
36+
seqan::hibf::empty_bin_fraction const empty_bin_fraction,
37+
seqan::hibf::track_occupancy const track_occupancy) :
3638
bins{bins_.value},
3739
bin_size_{size.value},
3840
hash_funs{funs.value},
39-
track_occupancy{track_occupancy_}
41+
track_occupancy{track_occupancy.value || empty_bin_fraction.value > 0.0}
4042
{
4143
if (bins == 0)
4244
throw std::logic_error{"The number of bins must be > 0."};
@@ -47,7 +49,14 @@ interleaved_bloom_filter::interleaved_bloom_filter(seqan::hibf::bin_count const
4749

4850
hash_shift = std::countl_zero(bin_size_);
4951
bin_words = divide_and_ceil(bins, 64u);
50-
technical_bins = bin_words * 64u;
52+
// Note: If the IBF is constructed as part of an HIBF, the layout only contains information about the
53+
// non-empty bins. Hence, we need to add the empty bins to the total number of bins.
54+
// Example: tmax = 320, empty_bin_fraction = 0.2
55+
// user bins = 320 * (1 - 0.2) = 256
56+
// The layout only contains information about the 256 bins.
57+
// The HIBF construction will then request an IBF with 256 bins.
58+
size_t const user_and_empty_bins = add_empty_bins(bins, empty_bin_fraction.value);
59+
technical_bins = next_multiple_of_64(user_and_empty_bins);
5160
resize(technical_bins * bin_size_);
5261
occupancy.resize(technical_bins, 0u);
5362
}
@@ -105,7 +114,8 @@ interleaved_bloom_filter::interleaved_bloom_filter(config & configuration, size_
105114
interleaved_bloom_filter{seqan::hibf::bin_count{configuration.number_of_user_bins},
106115
seqan::hibf::bin_size{max_bin_size(configuration, max_bin_elements)},
107116
seqan::hibf::hash_function_count{configuration.number_of_hash_functions},
108-
configuration.track_occupancy}
117+
seqan::hibf::empty_bin_fraction{configuration.empty_bin_fraction},
118+
seqan::hibf::track_occupancy{configuration.track_occupancy}}
109119
{
110120
size_t const chunk_size = std::clamp<size_t>(std::bit_ceil(bin_count() / configuration.threads), 8u, 64u);
111121

@@ -150,12 +160,13 @@ void interleaved_bloom_filter::clear(bin_index const bin) noexcept
150160
bool interleaved_bloom_filter::try_increase_bin_number_to(seqan::hibf::bin_count const new_bin_count) noexcept
151161
{
152162
size_t const new_bins = new_bin_count.value;
153-
size_t const new_bin_words = divide_and_ceil(new_bins, 64u);
163+
size_t const new_technical_bins = next_multiple_of_64(new_bins);
154164

155-
if (new_bins < bins || new_bin_words > bin_words)
165+
if (new_bins < bins || new_technical_bins > technical_bins)
156166
return false;
157167

158168
bins = new_bins;
169+
bin_words = divide_and_ceil(new_bins, 64u);
159170
return true;
160171
}
161172

‎src/layout/hierarchical_binning.cpp‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <hibf/layout/hierarchical_binning.hpp> // for hierarchical_binning
1818
#include <hibf/layout/layout.hpp> // for layout
1919
#include <hibf/layout/simple_binning.hpp> // for simple_binning
20+
#include <hibf/misc/add_empty_bins.hpp> // for add_empty_bins
2021
#include <hibf/misc/divide_and_ceil.hpp> // for divide_and_ceil
2122
#include <hibf/misc/next_multiple_of_64.hpp> // for next_multiple_of_64
2223
#include <hibf/misc/subtract_empty_bins.hpp> // for subtract_empty_bins
@@ -411,7 +412,7 @@ size_t hierarchical_binning::add_lower_level(data_store & libf_data) const
411412
size_t const number_of_user_bins = libf_data.positions.size();
412413

413414
// now do the binning for the low-level IBF:
414-
if (number_of_user_bins > config.tmax)
415+
if (number_of_user_bins > subtract_empty_bins(config.tmax, config.empty_bin_fraction))
415416
{
416417
// recursively call hierarchical binning if there are still too many UBs
417418
return hierarchical_binning{libf_data, config}.execute(); // return id of maximum technical bin
@@ -420,8 +421,13 @@ size_t hierarchical_binning::add_lower_level(data_store & libf_data) const
420421
{
421422
// use simple binning to distribute remaining UBs
422423
// Simple binning is not bound by config.tmax
423-
size_t const num_user_bins_next_64 = next_multiple_of_64(number_of_user_bins);
424-
size_t const number_of_technical_bins = subtract_empty_bins(num_user_bins_next_64, config.empty_bin_fraction);
424+
size_t const user_and_empty_bins = add_empty_bins(number_of_user_bins, config.empty_bin_fraction);
425+
// The empty-bin adjustment can, for some (tmax, empty_bin_fraction, number_of_user_bins) combinations,
426+
// round down to fewer technical bins than there are user bins. simple_binning requires
427+
// number_of_technical_bins >= number_of_user_bins, so never reserve fewer than that.
428+
size_t const number_of_technical_bins =
429+
std::max(number_of_user_bins,
430+
subtract_empty_bins(next_multiple_of_64(user_and_empty_bins), config.empty_bin_fraction));
425431
return simple_binning{libf_data, number_of_technical_bins}.execute(); // return id of maximum technical bin
426432
}
427433
}

‎test/performance/ibf/interleaved_bloom_filter_benchmark.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ inline void emplace_benchmark_impl(::benchmark::State & state)
104104
seqan::hibf::interleaved_bloom_filter ibf{seqan::hibf::bin_count{original_ibf.bin_count()},
105105
seqan::hibf::bin_size{original_ibf.bin_size()},
106106
seqan::hibf::hash_function_count{original_ibf.hash_function_count()},
107-
track_occupancy};
107+
seqan::hibf::empty_bin_fraction{0.0},
108+
seqan::hibf::track_occupancy{track_occupancy}};
108109

109110
for (auto _ : state)
110111
{

‎test/unit/hibf/config_test.cpp‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@
1111
#include <string> // for allocator, char_traits, string
1212
#include <utility> // for move
1313

14-
#include <hibf/config.hpp> // for config, insert_iterator
15-
#include <hibf/test/cereal.hpp> // for test_serialisation
16-
#include <hibf/test/expect_throw_msg.hpp> // for EXPECT_THROW_MSG
14+
#include <hibf/config.hpp> // for config, insert_iterator
15+
#include <hibf/misc/add_empty_bins.hpp> // for add_empty_bins
16+
#include <hibf/misc/subtract_empty_bins.hpp> // for subtract_empty_bins
17+
#include <hibf/test/cereal.hpp> // for test_serialisation
18+
#include <hibf/test/expect_throw_msg.hpp> // for EXPECT_THROW_MSG
1719

1820
TEST(config_test, write_to)
1921
{
@@ -466,6 +468,45 @@ TEST(config_test, validate_and_set_defaults)
466468
}
467469
}
468470

471+
TEST(config_test, empty_bin_fraction)
472+
{
473+
seqan::hibf::config config{.input_fn = [](size_t const, seqan::hibf::insert_iterator) {},
474+
.number_of_user_bins = 1u};
475+
476+
for (size_t top_level_tmax = 64; top_level_tmax <= 1024; top_level_tmax += 64)
477+
{
478+
config.tmax = top_level_tmax;
479+
480+
// Some deviation for high fractions
481+
for (double fraction = 0.00; fraction <= 0.97; fraction += 0.01)
482+
{
483+
size_t const top_level_subtracted = seqan::hibf::subtract_empty_bins(top_level_tmax, fraction);
484+
double const adjusted_fraction = 1.0 - static_cast<double>(top_level_subtracted) / top_level_tmax;
485+
486+
config.empty_bin_fraction = fraction;
487+
config.validate_and_set_defaults();
488+
EXPECT_NEAR(config.empty_bin_fraction, adjusted_fraction, 0.000000000000001);
489+
490+
// Roundtrip test
491+
for (size_t lower_level_tmax = 64; lower_level_tmax <= top_level_tmax; lower_level_tmax += 64)
492+
{
493+
size_t const lower_level_subtracted =
494+
seqan::hibf::subtract_empty_bins(lower_level_tmax, adjusted_fraction);
495+
size_t const lower_level_added = seqan::hibf::add_empty_bins(lower_level_subtracted, adjusted_fraction);
496+
497+
ASSERT_EQ(seqan::hibf::next_multiple_of_64(lower_level_added), lower_level_tmax)
498+
<< "top_level_tmax: " << top_level_tmax << '\n'
499+
<< "lower_level_tmax: " << lower_level_tmax << '\n'
500+
<< "fraction: " << fraction << '\n'
501+
<< "top_level_subtracted: " << top_level_subtracted << '\n'
502+
<< "adjusted_fraction: " << adjusted_fraction << '\n'
503+
<< "lower_level_subtracted: " << lower_level_subtracted << '\n'
504+
<< "lower_level_added: " << lower_level_added << '\n';
505+
}
506+
}
507+
}
508+
}
509+
469510
TEST(config_test, serialisation)
470511
{
471512
seqan::hibf::config config{.number_of_user_bins = 123456789,

0 commit comments

Comments
 (0)